From bd76a247ede1c25e1cf5d053509eefa92a5eaa26 Mon Sep 17 00:00:00 2001 From: Eric Promislow Date: Wed, 25 Sep 2024 11:14:23 -0700 Subject: [PATCH] make charts --- ...r-monitoring-crd-104.1.2-rc.1+up57.0.3.tgz | Bin 0 -> 304953 bytes ...ncher-monitoring-104.1.2-rc.1+up57.0.3.tgz | Bin 0 -> 478538 bytes .../104.1.2-rc.1+up57.0.3/Chart.yaml | 10 + .../104.1.2-rc.1+up57.0.3/README.md | 24 + .../files/crd-manifest.tgz | Bin 0 -> 308573 bytes .../templates/_helpers.tpl | 30 + .../104.1.2-rc.1+up57.0.3/templates/jobs.yaml | 102 + .../templates/manifest.yaml | 8 + .../104.1.2-rc.1+up57.0.3/templates/rbac.yaml | 76 + .../templates/validate-psp-install.yaml | 7 + .../104.1.2-rc.1+up57.0.3/values.yaml | 17 + .../104.1.2-rc.1+up57.0.3/.editorconfig | 5 + .../104.1.2-rc.1+up57.0.3/.helmignore | 29 + .../104.1.2-rc.1+up57.0.3/CHANGELOG.md | 47 + .../104.1.2-rc.1+up57.0.3/CONTRIBUTING.md | 12 + .../104.1.2-rc.1+up57.0.3/Chart.yaml | 126 + .../104.1.2-rc.1+up57.0.3/README.md | 1080 ++++ .../104.1.2-rc.1+up57.0.3/app-README.md | 46 + .../charts/grafana/.helmignore | 23 + .../charts/grafana/Chart.yaml | 39 + .../charts/grafana/README.md | 770 +++ .../grafana/dashboards/custom-dashboard.json | 1 + .../charts/grafana/templates/NOTES.txt | 55 + .../charts/grafana/templates/_config.tpl | 171 + .../charts/grafana/templates/_helpers.tpl | 305 + .../charts/grafana/templates/_pod.tpl | 1296 ++++ .../charts/grafana/templates/clusterrole.yaml | 25 + .../grafana/templates/clusterrolebinding.yaml | 24 + .../grafana/templates/configSecret.yaml | 43 + .../configmap-dashboard-provider.yaml | 15 + .../charts/grafana/templates/configmap.yaml | 15 + .../templates/dashboards-json-configmap.yaml | 38 + .../charts/grafana/templates/deployment.yaml | 53 + .../grafana/templates/extra-manifests.yaml | 4 + .../grafana/templates/headless-service.yaml | 22 + .../charts/grafana/templates/hpa.yaml | 52 + .../templates/image-renderer-deployment.yaml | 131 + .../grafana/templates/image-renderer-hpa.yaml | 47 + .../image-renderer-network-policy.yaml | 79 + .../templates/image-renderer-service.yaml | 31 + .../image-renderer-servicemonitor.yaml | 48 + .../charts/grafana/templates/ingress.yaml | 78 + .../grafana/templates/networkpolicy.yaml | 61 + .../grafana/templates/nginx-config.yaml | 94 + .../templates/poddisruptionbudget.yaml | 22 + .../grafana/templates/podsecuritypolicy.yaml | 45 + .../charts/grafana/templates/pvc.yaml | 41 + .../charts/grafana/templates/role.yaml | 32 + .../charts/grafana/templates/rolebinding.yaml | 25 + .../charts/grafana/templates/secret-env.yaml | 14 + .../charts/grafana/templates/secret.yaml | 16 + .../charts/grafana/templates/service.yaml | 61 + .../grafana/templates/serviceaccount.yaml | 17 + .../grafana/templates/servicemonitor.yaml | 68 + .../charts/grafana/templates/statefulset.yaml | 58 + .../templates/tests/test-configmap.yaml | 20 + .../tests/test-podsecuritypolicy.yaml | 32 + .../grafana/templates/tests/test-role.yaml | 17 + .../templates/tests/test-rolebinding.yaml | 20 + .../templates/tests/test-serviceaccount.yaml | 12 + .../charts/grafana/templates/tests/test.yaml | 53 + .../charts/grafana/values.yaml | 1315 ++++ .../charts/hardenedKubelet/.helmignore | 23 + .../charts/hardenedKubelet/Chart.yaml | 15 + .../charts/hardenedKubelet/README.md | 90 + .../hardenedKubelet/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedKubelet/values.yaml | 166 + .../charts/hardenedNodeExporter/.helmignore | 23 + .../charts/hardenedNodeExporter/Chart.yaml | 15 + .../charts/hardenedNodeExporter/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedNodeExporter/values.yaml | 166 + .../charts/k3sServer/.helmignore | 23 + .../charts/k3sServer/Chart.yaml | 15 + .../charts/k3sServer/README.md | 90 + .../charts/k3sServer/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../k3sServer/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../k3sServer/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/k3sServer/values.yaml | 166 + .../charts/kube-state-metrics/.helmignore | 21 + .../charts/kube-state-metrics/Chart.yaml | 32 + .../charts/kube-state-metrics/README.md | 85 + .../kube-state-metrics/templates/NOTES.txt | 23 + .../kube-state-metrics/templates/_helpers.tpl | 196 + .../templates/ciliumnetworkpolicy.yaml | 33 + .../templates/clusterrolebinding.yaml | 20 + .../templates/crs-configmap.yaml | 16 + .../templates/deployment.yaml | 314 + .../templates/extra-manifests.yaml | 4 + .../templates/kubeconfig-secret.yaml | 12 + .../templates/networkpolicy.yaml | 43 + .../kube-state-metrics/templates/pdb.yaml | 18 + .../templates/podsecuritypolicy.yaml | 39 + .../templates/psp-clusterrole.yaml | 19 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/rbac-configmap.yaml | 22 + .../kube-state-metrics/templates/role.yaml | 215 + .../templates/rolebinding.yaml | 24 + .../kube-state-metrics/templates/service.yaml | 49 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 126 + .../templates/stsdiscovery-role.yaml | 26 + .../templates/stsdiscovery-rolebinding.yaml | 17 + .../templates/verticalpodautoscaler.yaml | 44 + .../charts/kube-state-metrics/values.yaml | 491 ++ .../kubeAdmControllerManager/.helmignore | 23 + .../kubeAdmControllerManager/Chart.yaml | 15 + .../charts/kubeAdmControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../kubeAdmControllerManager/values.yaml | 166 + .../charts/kubeAdmEtcd/.helmignore | 23 + .../charts/kubeAdmEtcd/Chart.yaml | 15 + .../charts/kubeAdmEtcd/README.md | 90 + .../charts/kubeAdmEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../kubeAdmEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmEtcd/values.yaml | 166 + .../charts/kubeAdmProxy/.helmignore | 23 + .../charts/kubeAdmProxy/Chart.yaml | 15 + .../charts/kubeAdmProxy/README.md | 90 + .../kubeAdmProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmProxy/values.yaml | 166 + .../charts/kubeAdmScheduler/.helmignore | 23 + .../charts/kubeAdmScheduler/Chart.yaml | 15 + .../charts/kubeAdmScheduler/README.md | 90 + .../kubeAdmScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmScheduler/values.yaml | 166 + .../charts/prometheus-adapter/.helmignore | 21 + .../charts/prometheus-adapter/Chart.yaml | 28 + .../charts/prometheus-adapter/README.md | 160 + .../prometheus-adapter/templates/NOTES.txt | 9 + .../prometheus-adapter/templates/_helpers.tpl | 113 + .../templates/certmanager.yaml | 76 + .../cluster-role-binding-auth-delegator.yaml | 20 + .../cluster-role-binding-resource-reader.yaml | 20 + .../cluster-role-resource-reader.yaml | 24 + .../templates/configmap.yaml | 97 + .../templates/custom-metrics-apiservice.yaml | 34 + ...stom-metrics-cluster-role-binding-hpa.yaml | 24 + .../custom-metrics-cluster-role.yaml | 17 + .../templates/deployment.yaml | 143 + .../external-metrics-apiservice.yaml | 34 + ...rnal-metrics-cluster-role-binding-hpa.yaml | 20 + .../external-metrics-cluster-role.yaml | 21 + .../prometheus-adapter/templates/pdb.yaml | 23 + .../prometheus-adapter/templates/psp.yaml | 66 + .../resource-metrics-apiservice.yaml | 34 + ...resource-metrics-cluster-role-binding.yaml | 20 + .../resource-metrics-cluster-role.yaml | 23 + .../templates/role-binding-auth-reader.yaml | 21 + .../prometheus-adapter/templates/secret.yaml | 17 + .../prometheus-adapter/templates/service.yaml | 27 + .../templates/serviceaccount.yaml | 18 + .../charts/prometheus-adapter/values.yaml | 277 + .../prometheus-node-exporter/.helmignore | 21 + .../prometheus-node-exporter/Chart.yaml | 25 + .../charts/prometheus-node-exporter/README.md | 97 + .../templates/NOTES.txt | 29 + .../templates/_helpers.tpl | 236 + .../templates/clusterrole.yaml | 19 + .../templates/clusterrolebinding.yaml | 20 + .../templates/daemonset.yaml | 309 + .../templates/endpoints.yaml | 18 + .../templates/extra-manifests.yaml | 4 + .../templates/networkpolicy.yaml | 23 + .../templates/podmonitor.yaml | 91 + .../templates/psp-clusterrole.yaml | 14 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/psp.yaml | 49 + .../templates/rbac-configmap.yaml | 16 + .../templates/service.yaml | 29 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 71 + .../templates/verticalpodautoscaler.yaml | 40 + .../prometheus-node-exporter/values.yaml | 530 ++ .../charts/rke2ControllerManager/.helmignore | 23 + .../charts/rke2ControllerManager/Chart.yaml | 15 + .../charts/rke2ControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2ControllerManager/values.yaml | 166 + .../charts/rke2Etcd/.helmignore | 23 + .../charts/rke2Etcd/Chart.yaml | 15 + .../charts/rke2Etcd/README.md | 90 + .../charts/rke2Etcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Etcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Etcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Etcd/values.yaml | 166 + .../charts/rke2IngressNginx/.helmignore | 23 + .../charts/rke2IngressNginx/Chart.yaml | 15 + .../charts/rke2IngressNginx/README.md | 90 + .../rke2IngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2IngressNginx/values.yaml | 166 + .../charts/rke2Proxy/.helmignore | 23 + .../charts/rke2Proxy/Chart.yaml | 15 + .../charts/rke2Proxy/README.md | 90 + .../charts/rke2Proxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Proxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Proxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Proxy/values.yaml | 166 + .../charts/rke2Scheduler/.helmignore | 23 + .../charts/rke2Scheduler/Chart.yaml | 15 + .../charts/rke2Scheduler/README.md | 90 + .../rke2Scheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Scheduler/values.yaml | 166 + .../charts/rkeControllerManager/.helmignore | 23 + .../charts/rkeControllerManager/Chart.yaml | 15 + .../charts/rkeControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeControllerManager/values.yaml | 166 + .../charts/rkeEtcd/.helmignore | 23 + .../charts/rkeEtcd/Chart.yaml | 15 + .../charts/rkeEtcd/README.md | 90 + .../charts/rkeEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeEtcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeEtcd/values.yaml | 166 + .../charts/rkeIngressNginx/.helmignore | 23 + .../charts/rkeIngressNginx/Chart.yaml | 15 + .../charts/rkeIngressNginx/README.md | 90 + .../rkeIngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeIngressNginx/values.yaml | 166 + .../charts/rkeProxy/.helmignore | 23 + .../charts/rkeProxy/Chart.yaml | 15 + .../charts/rkeProxy/README.md | 90 + .../charts/rkeProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeProxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeProxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeProxy/values.yaml | 166 + .../charts/rkeScheduler/.helmignore | 23 + .../charts/rkeScheduler/Chart.yaml | 15 + .../charts/rkeScheduler/README.md | 90 + .../rkeScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeScheduler/values.yaml | 166 + .../charts/windowsExporter/.helmignore | 21 + .../charts/windowsExporter/Chart.yaml | 17 + .../charts/windowsExporter/README.md | 42 + .../scripts/configure-firewall.ps1 | 31 + .../windowsExporter/templates/_helpers.tpl | 216 + .../windowsExporter/templates/config.yaml | 14 + .../windowsExporter/templates/daemonset.yaml | 200 + .../windowsExporter/templates/podmonitor.yaml | 91 + .../templates/scriptConfig.yaml | 14 + .../windowsExporter/templates/service.yaml | 32 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 75 + .../charts/windowsExporter/values.yaml | 366 ++ .../files/ingress-nginx/nginx.json | 1445 +++++ .../request-handling-performance.json | 963 +++ .../cluster/rancher-cluster-nodes.json | 793 +++ .../rancher/cluster/rancher-cluster.json | 776 +++ .../files/rancher/fleet/bundle.json | 246 + .../files/rancher/fleet/bundledeployment.json | 219 + .../files/rancher/fleet/cluster.json | 484 ++ .../files/rancher/fleet/clustergroup.json | 468 ++ .../rancher/fleet/controller-runtime.json | 454 ++ .../files/rancher/fleet/gitrepo.json | 325 + .../rancher/home/rancher-default-home.json | 1290 ++++ .../files/rancher/k8s/rancher-etcd-nodes.json | 687 +++ .../files/rancher/k8s/rancher-etcd.json | 669 ++ .../k8s/rancher-k8s-components-nodes.json | 527 ++ .../rancher/k8s/rancher-k8s-components.json | 519 ++ .../rancher/nodes/rancher-node-detail.json | 805 +++ .../files/rancher/nodes/rancher-node.json | 792 +++ .../performance/performance-debugging.json | 1652 +++++ .../rancher/pods/rancher-pod-containers.json | 636 ++ .../files/rancher/pods/rancher-pod.json | 636 ++ .../workloads/rancher-workload-pods.json | 652 ++ .../rancher/workloads/rancher-workload.json | 652 ++ .../delete-workloads-with-old-labels.sh | 14 + .../104.1.2-rc.1+up57.0.3/templates/NOTES.txt | 4 + .../templates/_helpers.tpl | 459 ++ .../templates/alertmanager/alertmanager.yaml | 191 + .../templates/alertmanager/extrasecret.yaml | 20 + .../templates/alertmanager/ingress.yaml | 78 + .../alertmanager/ingressperreplica.yaml | 67 + .../alertmanager/podDisruptionBudget.yaml | 21 + .../templates/alertmanager/psp-role.yaml | 23 + .../alertmanager/psp-rolebinding.yaml | 20 + .../templates/alertmanager/psp.yaml | 47 + .../templates/alertmanager/secret.yaml | 35 + .../templates/alertmanager/service.yaml | 68 + .../alertmanager/serviceaccount.yaml | 21 + .../alertmanager/servicemonitor.yaml | 84 + .../alertmanager/serviceperreplica.yaml | 49 + .../templates/exporters/core-dns/service.yaml | 24 + .../exporters/core-dns/servicemonitor.yaml | 58 + .../kube-api-server/servicemonitor.yaml | 57 + .../kube-controller-manager/endpoints.yaml | 22 + .../kube-controller-manager/service.yaml | 29 + .../servicemonitor.yaml | 69 + .../templates/exporters/kube-dns/service.yaml | 28 + .../exporters/kube-dns/servicemonitor.yaml | 71 + .../exporters/kube-etcd/endpoints.yaml | 20 + .../exporters/kube-etcd/service.yaml | 27 + .../exporters/kube-etcd/servicemonitor.yaml | 75 + .../exporters/kube-proxy/endpoints.yaml | 20 + .../exporters/kube-proxy/service.yaml | 27 + .../exporters/kube-proxy/servicemonitor.yaml | 63 + .../exporters/kube-scheduler/endpoints.yaml | 22 + .../exporters/kube-scheduler/service.yaml | 29 + .../kube-scheduler/servicemonitor.yaml | 69 + .../kube-state-metrics/validate.yaml | 7 + .../exporters/kubelet/servicemonitor.yaml | 246 + .../exporters/node-exporter/validate.yaml | 3 + .../templates/extra-objects.yaml | 4 + .../grafana/configmap-dashboards.yaml | 24 + .../grafana/configmaps-datasources.yaml | 81 + .../alertmanager-overview.yaml | 616 ++ .../grafana/dashboards-1.14/apiserver.yaml | 1772 ++++++ .../dashboards-1.14/cluster-total.yaml | 1882 ++++++ .../dashboards-1.14/controller-manager.yaml | 1196 ++++ .../grafana/dashboards-1.14/etcd.yaml | 1229 ++++ .../dashboards-1.14/grafana-overview.yaml | 635 ++ .../grafana/dashboards-1.14/k8s-coredns.yaml | 1534 +++++ .../k8s-resources-cluster.yaml | 3088 ++++++++++ .../k8s-resources-multicluster.yaml | 24 + .../k8s-resources-namespace.yaml | 2797 +++++++++ .../dashboards-1.14/k8s-resources-node.yaml | 1026 ++++ .../dashboards-1.14/k8s-resources-pod.yaml | 2469 ++++++++ .../k8s-resources-windows-cluster.yaml | 24 + .../k8s-resources-windows-namespace.yaml | 24 + .../k8s-resources-windows-pod.yaml | 24 + .../k8s-resources-workload.yaml | 2024 ++++++ .../k8s-resources-workloads-namespace.yaml | 2189 +++++++ .../k8s-windows-cluster-rsrc-use.yaml | 24 + .../k8s-windows-node-rsrc-use.yaml | 24 + .../grafana/dashboards-1.14/kubelet.yaml | 2256 +++++++ .../dashboards-1.14/namespace-by-pod.yaml | 1464 +++++ .../namespace-by-workload.yaml | 1736 ++++++ .../node-cluster-rsrc-use.yaml | 1063 ++++ .../dashboards-1.14/node-rsrc-use.yaml | 1089 ++++ .../grafana/dashboards-1.14/nodes-darwin.yaml | 1073 ++++ .../grafana/dashboards-1.14/nodes.yaml | 1066 ++++ .../persistentvolumesusage.yaml | 587 ++ .../grafana/dashboards-1.14/pod-total.yaml | 1228 ++++ .../prometheus-remote-write.yaml | 1674 +++++ .../grafana/dashboards-1.14/prometheus.yaml | 1235 ++++ .../grafana/dashboards-1.14/proxy.yaml | 1276 ++++ .../grafana/dashboards-1.14/scheduler.yaml | 1118 ++++ .../dashboards-1.14/workload-total.yaml | 1438 +++++ .../templates/grafana/namespaces.yaml | 13 + .../_prometheus-operator.tpl | 7 + .../_prometheus-operator-webhook.tpl | 6 + .../deployment/deployment.yaml | 143 + .../admission-webhooks/deployment/pdb.yaml | 15 + .../deployment/service.yaml | 58 + .../deployment/serviceaccount.yaml | 15 + .../ciliumnetworkpolicy-createSecret.yaml | 36 + .../ciliumnetworkpolicy-patchWebhook.yaml | 36 + .../job-patch/clusterrole.yaml | 33 + .../job-patch/clusterrolebinding.yaml | 20 + .../job-patch/job-createSecret.yaml | 73 + .../job-patch/job-patchWebhook.yaml | 74 + .../job-patch/networkpolicy-createSecret.yaml | 33 + .../job-patch/networkpolicy-patchWebhook.yaml | 33 + .../admission-webhooks/job-patch/psp.yaml | 47 + .../admission-webhooks/job-patch/role.yaml | 21 + .../job-patch/rolebinding.yaml | 21 + .../job-patch/serviceaccount.yaml | 17 + .../mutatingWebhookConfiguration.yaml | 77 + .../validatingWebhookConfiguration.yaml | 77 + .../prometheus-operator/certmanager.yaml | 55 + .../ciliumnetworkpolicy.yaml | 40 + .../prometheus-operator/clusterrole.yaml | 109 + .../clusterrolebinding.yaml | 16 + .../prometheus-operator/deployment.yaml | 204 + .../prometheus-operator/networkpolicy.yaml | 29 + .../prometheus-operator/psp-clusterrole.yaml | 21 + .../psp-clusterrolebinding.yaml | 18 + .../templates/prometheus-operator/psp.yaml | 46 + .../prometheus-operator/service.yaml | 57 + .../prometheus-operator/serviceaccount.yaml | 14 + .../prometheus-operator/servicemonitor.yaml | 57 + .../verticalpodautoscaler.yaml | 40 + .../templates/prometheus/_rules.tpl | 44 + .../additionalAlertRelabelConfigs.yaml | 16 + .../additionalAlertmanagerConfigs.yaml | 16 + .../prometheus/additionalPrometheusRules.yaml | 43 + .../prometheus/additionalScrapeConfigs.yaml | 20 + .../prometheus/ciliumnetworkpolicy.yaml | 27 + .../templates/prometheus/clusterrole.yaml | 30 + .../prometheus/clusterrolebinding.yaml | 18 + .../templates/prometheus/csi-secret.yaml | 12 + .../templates/prometheus/extrasecret.yaml | 20 + .../templates/prometheus/ingress.yaml | 77 + .../prometheus/ingressThanosSidecar.yaml | 77 + .../prometheus/ingressperreplica.yaml | 67 + .../templates/prometheus/networkpolicy.yaml | 34 + .../templates/prometheus/nginx-config.yaml | 68 + .../prometheus/podDisruptionBudget.yaml | 25 + .../templates/prometheus/podmonitors.yaml | 38 + .../templates/prometheus/prometheus.yaml | 472 ++ .../templates/prometheus/psp-clusterrole.yaml | 22 + .../prometheus/psp-clusterrolebinding.yaml | 19 + .../templates/prometheus/psp.yaml | 58 + .../rules-1.14/alertmanager.rules.yaml | 305 + .../rules-1.14/config-reloaders.yaml | 57 + .../templates/prometheus/rules-1.14/etcd.yaml | 461 ++ .../prometheus/rules-1.14/general.rules.yaml | 125 + ...les.container_cpu_usage_seconds_total.yaml | 43 + .../k8s.rules.container_memory_cache.yaml | 42 + .../k8s.rules.container_memory_rss.yaml | 42 + .../k8s.rules.container_memory_swap.yaml | 42 + ...es.container_memory_working_set_bytes.yaml | 42 + .../k8s.rules.container_resource.yaml | 168 + .../rules-1.14/k8s.rules.pod_owner.yaml | 107 + .../prometheus/rules-1.14/k8s.rules.yaml | 237 + .../kube-apiserver-availability.rules.yaml | 273 + .../kube-apiserver-burnrate.rules.yaml | 440 ++ .../kube-apiserver-histogram.rules.yaml | 53 + .../rules-1.14/kube-apiserver-slos.yaml | 159 + .../kube-prometheus-general.rules.yaml | 49 + .../kube-prometheus-node-recording.rules.yaml | 93 + .../rules-1.14/kube-scheduler.rules.yaml | 135 + .../rules-1.14/kube-state-metrics.yaml | 152 + .../prometheus/rules-1.14/kubelet.rules.yaml | 65 + .../rules-1.14/kubernetes-apps.yaml | 568 ++ .../rules-1.14/kubernetes-resources.yaml | 282 + .../rules-1.14/kubernetes-storage.yaml | 216 + .../kubernetes-system-apiserver.yaml | 193 + .../kubernetes-system-controller-manager.yaml | 55 + .../kubernetes-system-kube-proxy.yaml | 56 + .../rules-1.14/kubernetes-system-kubelet.yaml | 379 ++ .../kubernetes-system-scheduler.yaml | 54 + .../rules-1.14/kubernetes-system.yaml | 87 + .../rules-1.14/node-exporter.rules.yaml | 188 + .../prometheus/rules-1.14/node-exporter.yaml | 801 +++ .../prometheus/rules-1.14/node-network.yaml | 55 + .../prometheus/rules-1.14/node.rules.yaml | 109 + .../rules-1.14/prometheus-operator.yaml | 253 + .../prometheus/rules-1.14/prometheus.yaml | 707 +++ .../rules-1.14/windows.node.rules.yaml | 301 + .../rules-1.14/windows.pod.rules.yaml | 158 + .../templates/prometheus/secret.yaml | 15 + .../templates/prometheus/service.yaml | 80 + .../prometheus/serviceThanosSidecar.yaml | 39 + .../serviceThanosSidecarExternal.yaml | 46 + .../templates/prometheus/serviceaccount.yaml | 21 + .../templates/prometheus/servicemonitor.yaml | 97 + .../servicemonitorThanosSidecar.yaml | 55 + .../templates/prometheus/servicemonitors.yaml | 47 + .../prometheus/serviceperreplica.yaml | 54 + .../rancher-monitoring/clusterrole.yaml | 135 + .../rancher-monitoring/config-role.yaml | 48 + .../rancher-monitoring/dashboard-role.yaml | 47 + .../addons/ingress-nginx-dashboard.yaml | 18 + .../rancher/cluster-dashboards.yaml | 17 + .../dashboards/rancher/default-dashboard.yaml | 17 + .../dashboards/rancher/fleet-dashboards.yaml | 17 + .../dashboards/rancher/k8s-dashboards.yaml | 31 + .../dashboards/rancher/nodes-dashboards.yaml | 17 + .../rancher/performance-dashboards.yaml | 18 + .../dashboards/rancher/pods-dashboards.yaml | 17 + .../rancher/workload-dashboards.yaml | 17 + .../exporters/fleet/servicemonitor.yaml | 53 + .../exporters/ingress-nginx/service.yaml | 27 + .../ingress-nginx/servicemonitor.yaml | 49 + .../exporters/rancher/servicemonitor.yaml | 58 + .../rancher-monitoring/hardened.yaml | 147 + .../rancher-monitoring/upgrade/configmap.yaml | 13 + .../rancher-monitoring/upgrade/job.yaml | 46 + .../rancher-monitoring/upgrade/rbac.yaml | 131 + .../templates/thanos-ruler/extrasecret.yaml | 20 + .../templates/thanos-ruler/ingress.yaml | 77 + .../thanos-ruler/podDisruptionBudget.yaml | 21 + .../templates/thanos-ruler/ruler.yaml | 189 + .../templates/thanos-ruler/secret.yaml | 26 + .../templates/thanos-ruler/service.yaml | 53 + .../thanos-ruler/serviceaccount.yaml | 20 + .../thanos-ruler/servicemonitor.yaml | 82 + .../templates/validate-install-crd.yaml | 23 + .../templates/validate-psp-install.yaml | 7 + .../104.1.2-rc.1+up57.0.3/values.yaml | 5431 +++++++++++++++++ index.yaml | 144 + 581 files changed, 105235 insertions(+) create mode 100644 assets/rancher-monitoring-crd/rancher-monitoring-crd-104.1.2-rc.1+up57.0.3.tgz create mode 100644 assets/rancher-monitoring/rancher-monitoring-104.1.2-rc.1+up57.0.3.tgz create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/Chart.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/README.md create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/files/crd-manifest.tgz create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/jobs.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/manifest.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/rbac.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.editorconfig create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CHANGELOG.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CONTRIBUTING.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/app-README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/dashboards/custom-dashboard.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_config.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_pod.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/headless-service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/nginx-config.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/pvc.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret-env.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/statefulset.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/config.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/nginx.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/request-handling-performance.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundle.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundledeployment.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/cluster.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/clustergroup.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/controller-runtime.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/gitrepo.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/home/rancher-default-home.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node-detail.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/performance/performance-debugging.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod-containers.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload-pods.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/alertmanager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/extrasecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/node-exporter/validate.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/extra-objects.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmap-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmaps-datasources.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/namespaces.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/certmanager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/_rules.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/csi-secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/extrasecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/nginx-config.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podmonitors.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/prometheus.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitors.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/config-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/hardened.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/extrasecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ruler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/values.yaml diff --git a/assets/rancher-monitoring-crd/rancher-monitoring-crd-104.1.2-rc.1+up57.0.3.tgz b/assets/rancher-monitoring-crd/rancher-monitoring-crd-104.1.2-rc.1+up57.0.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..17edabb7ea1392508eeeff85fbd5ec318176e814 GIT binary patch literal 304953 zcmV(^K-Iq=iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMR7Jd|JCI8OFKwuB_Nofup8?7Ng?D_O_PjbY4;S*+QKh)O~s zOURyNDMDGwk}X=1tz-!$B$EGUhPLnX-QM^2{GR9c{@>?(`slvTx$f&+=eo|do-;TY z8tx9_pk5d>l7PV>(QZ&U4k4rG4#N?o{b61xivRN_D=RCjq@Y0lUshK3xBts39#Ei= zQ&3V?lvR{Fpr}Y8E2pTaBu4?r{uAc@%SOZ#U^oid|KquSySV=$4TeTz2rvQ?gT|{t z000gnz)%=BX*i5PK!MUoj0_ya5sC4K*$h~I2j2n>s~0daUF2CWA8$U_hy9*#p| z3FLPZG@bxMq3{5~9R&0&^zncz2KPINq#=Jg1;HN+ssS)87KMb9IR^3hHlv)Zg0!5p zJQN3)mXjo66_urBr4K;<-Lm~V?GN~k5DySAfC~sC42yL$QkGb|2?Cb$BEB6u0W z{}uuejrfOWh!AibiAG?2@qi@}i^bpwq{YbWhkWaWqlkC{hy#3)C=>ugAb{<40R$Kl zO(2^*6cSDJ14woNF9jiixPT~(?{@c3#(-qbd@(pr6b6RC18@uxMFIuzaCZ zIKdr*2Y<#0*)KyP^JfN=N1>!q=ugzeiq1Cp(C3S$Vt{|MC z1_+vAEVl_Dbz`t3d?ZUIXy8qRp#*>B)*q08-?o3h$QTcL`G7b*j2D(<)CJY$q~#Q# zvVwrL9t;a}L86cZB#4(bCAxsy(U-IllCb?q#^VAKVA7sM7Z8UANyeVM7_}|tY>|b+ zgJj4*1^PXy|GQP^{}=5q?Y{@c<^MDe{0sZ9bU;!5H}Rjcf}-+&?Ek-_{hT+szzA@6 z86P=_ClZZN0}f$aAYLE=hJX=Zq-bg@ISLE}q_?UAW-u=h2nvG2aUhHUZUby5cqAx` zt-Os4NA3oKf*=5Zg1LYw{8nTK!(x92BLo0`la>O2pgV~2lE%BsxMMJ$f@%O3gC{_d zTZx_2)*BJ)hJzu%?Kd0-g>r$xJ^%4*C;~)*1Q3eFppbBXK{dbybj9F6D0z&$7$gQQ z1(09yL^vD-5g=mw-4%vJfd~j53&KfseN&Sy8h>UMsr@CxNbP5K{3%NS;M=U^XAtLu zgo8S8IEIKOnEk;_UNAJwjhwM>eFG=rkOY4{44MG?5q`{xBcgTiW*D>u21EEc#{NEL zjR$dRfb4cB9D^n~P7wDK+0gCPZmo@EL-60n0AMR3ClUCKV}BB2NCQDZq@`>}2jp+1 ze}wWtP>|X`3=?2(-?;SCjOIiX%A71^HNeEx3_~!-fp`#2_z4b<@$!P95o$kmK>-;T zBw7aV{!1Se{>!W8Pj3JK!rd`|pbi#`@<*cG$TbFOX=%ZOzW{l8B9J%$iUnjq0$c`; zL;O1I0>gt!3IG(bJ&r-UBHc(}Bq<_YK|Dd4;1(e5qND(v0thf10QC#-0ib$+fR5x& z;RF;Z2SSl_4{G819TGTo>ihVXzxc!i-gl0MsQdAnG=w2Uy|%K;hR*Cq77k46cP6g$8Qq3 zKmdt`qlkzn_{U*Tu8R@S;1dJ`r3-$Yee~6hBzme>VdE{Tjeuo&Ow=m-}t~_kf(zfAar-Mf>I4^jnCf z_f5h6R-pBhe*T?M>l?b?^Dv}-l4+1OM4~{vv=K-E1YJpw|KN`Vfm48sk^+c^1E+v( z`=UTX|Lpxl4Fe1SngF^<8y|=Wd z3b`8p!!m50q*!23;NP_k+ljzm=U3l>aYPg;5`Y3QEYb*vA!5m&kzYYS0*EG2i{B=V ze9DW*5OHvD8wyUu6EI%iI>`qeNOB|s0RSHm=Ym%Q?8LUJ9%52}m>WnS{f8wbnWTrU zxRvxo#3Hs~P)IyM%;C?^y|8$!^!70|Kt99yK|Ek5ww3dVIs8H{xgYT@J0gQ1K|C2~ zn^01-6C>imU%n)p)2+q+F%y4^uzs6R|JUTKi$o)kXt)21;>Msr3(%DW`;BLRLL&qK z{&6e*HAQ%$iw6iNkhj&=_m;n8vVUt)f5*N3mf7D~`kVRwYwY}I)iA!JB$4@F+0Q>O zX=^jf_{h0{1lYIbnPU)^+dFwHi~4u3@LSaai$nS#QJ~w;*GFI|6vo&5+kgQc4nx7V z?uvbbaL3>YW+1^AgY*2g)5Kiw*A8Y)9_AOL^|50g@` zKOQGyu_(|BL=#}BAI9%H=n*2GKn54v>V=`*e$JY_kZ3hP?t5C~1@lt_loS;YC_(_h z6~Coh|0HzZl5GnNhCmM2$aR$;toS9Pg92jTvo^7BJ$Mj~0|{b3o2q}eJN#eQ{xbga zfuWEjKM2L*vEL5d{)_S8U*&&FvcI|iCoiw4@SpoX|B@!O?F8HifH(?71L7zQ#*>Hz z1ph2Z5CjB&^1gzAAnBik1RzVkm?89E@hjhe(HH_C{_Cw09dnaymxGr!hT#FhU;U2E zKX@ELiSHO)VMr7pXypz91Q^Z@Bz(UDgusA!@?`EXACP<@_#e;!I_4(dV=ZaGoa~4I z1QJgo8Zfss#{)<-KyXLm0XWI?Y}fGtEDi$)K~k(FE%<{G+lK9rp8wtQ{{#KEFwoII zVj%5>_-D-V7y7RtFaO*8Pf_ka^`C!D6Z)gu?mFKt25xV1f;)@=AphYeJrNHg02lw? zy-g1h5(11dzFRu?lfICGDlG`a>PL?k42O3oBZi?+0LJwje)6?qJU~9dMr>&v2sr;- z3vvb!NF2E`?!R@p81|!jwM{er8`&TNz-)nhvj#8}2JMDNB0vBKqCg)QngCen>gWOD zzXt14zmpIB(kBJL(1?GuBofjP$Wb(W%g%hC0Rdvct%B7zBLm_A7!Lf2Oc`MNG7ISu zhava5z~G)zKeG|^1K~sh>Dym!!vHeBT!;Dg`t3dpa0ufHQD#`vNQ zksu0z*C5A=&Ih+vg(rYmlC*H3HxY>g5da>EhJ#Y%11gf_06Wv~R|EkB299?ScP9|A zcr_WBU+)f*#$*UEyeD3UTrWUCe~im)l?R~Tv+&<`K!2(j$OyTT=!9Z%L^KlZCP9`m z46q|FfxL}jI1CYu_(6yplHg8sk%nWuexeDAN4nuco_v{ zB_Z-79OFd_`=Lsza*A@Qst1%LqyRh-?!GMm^4T-s0{++?u1aBch z{<29(`T@Yu0Exfbce#FlhvPvk32m z*&RfA{gAS=)KAdcQ=stVABx~WFN_a}_=jcwWUGHrx8-AwyZe*WLyQ3Uf`A7RPuOx8 zKUlWC>K{qXPflZdC0;NjdJCr(1Y&{)j7jtoFbJ5x6yT2`ep5fh_Rm{jWbZ^W2S0kg zS&%K1FtP;1@Z@R}nSvj)lO&HJ5&#fQb}t~wT>>z2&`QLCB#z-o9s;+uJUcQi2OR$P zI^j6PUoMdp8gJnxZ-2bZ&yEy>M){Kq5f~g{%hzwqjhqLN@R6pI0$hlM9|83?JQ@6( z#*-LC7AOq$<0~Q>h4cggUw88E`@Yd!VQ_NfPyS8NzdJ1YU(@~|{{Q_$xBuoGe-{7C z$sLeaJfQg7^Ius-+5g1<|BA*RN5A#3`L5p6ccQvVcBtnwr=O^8=YitncbDx0X7DfX z&ig%HYq!6P67#RSqIBBWSoa9K6h+6itLYyHGoyG74hd7XJ#OiKw;}N1`RmCCUL!Ys zdwuDl*TF)=YQ!pEPs8M2)Bb@M}<%%e@_FO#3SoHph=CuA~1zP?FmR1N<6 zIpv|#7uujZ%CURQ8P0shB%EWoZld}={k)$V-)bFi%=)HK*n_p=&9hj;+mF1%+#TUpqgO5`|WDfOXUjD9S~nZ4MFartQE09LhAzRBo- z#aT=-!(}T4Q-)`4!`HbUPQ1-LcWzk{Z9MTr)~@GD`zm$zj<>I=iOe_ff|_h1CcCT; zbQdjB6SWY%YBM*54?HMb|3ptbT>mxh^0j_isEi_X?Hmv1;^0*}M@xZaJ z>F|gr1#`SIp0)d8;pfn-){+j{g@ARANd2ZmxbUS;5i{d823;RD#fl3V9j*BEOZYEC zqlYJjA5Pi@9E8zEhmFL-C|*!80Q^- z>$S%VM12nJp6gj32CMG#6?{m$ZeCQ~$d{O5lK4_W4;A}59nxabeg=qAWremFYeX`( zh;9m>*?Ud0iF1!~MlgPL*>{j^Kx(CpKsR5${QfG!$B(tLUR(7~?st)ugPKb=_ z2hms?P-O<|pY7v)z>w3Llrg~MX_$9#gQ4?dQ5McWmqk9qD$(0JDUqf$yx1P&-j&YJzb5M zDA1U#YTsFQDDgSXz}pPbHB_pQ-oRjz=gL4N!{rGYa7g2czTHQzc(=1(D;3-<53?jfC0{)L=Z zi^esU525zMC5J+qWg9y84}KYZ$u9mOrvc@3toij!m0ayrj*mjM*KDizd|@}K4&%7f z-gfiw>O{P6;rP8YL>XH^!yd~n2k)C^%87U>wr+W!n&4b!qB^fi9B!jd)<>$d2)>X5uC_&KrWLFIcb2wEjz*+ zp3pw1p}sI})n2gAzP{M=*|oGzlxDcYOhM>{ z2S0yK+M4mOLP^%4^SksM}lFk+0CXbh30cxMn{5>{oe#lTjwo`kndoJiC;D zliHQfjkGIUE<$oZza2W2DD}r;Cu!N18jGl#6ua8}1sI%0r3T)G+*)yL?(#svp?oJq z&V1DH>9uFfg&tYoCsl~wxMp?bt9c~ zX+n8Q*-i}t!orjC0ZYecN}cYnQt4ZLydTWlW>K_K=+<0yr#?74Nb{4A0+m@FUzl(5 zT!#$9WIWu2aVf<67_5OnWs1#iNw^4e$hppZF)vwhN`j}SH>}m?{o&1q4u1kI^_aJz?9Imdws*OIQJ4u3wrd@9hh$Y~Dl^we)%IWJ$+?~UV^ z)UTi3Zhl;vy~y#gpYG~Arw+Xq6WytB0u+DUtspqq?nZZ>fq5}q7ZBKQ4S%*3GP{G$99LpCm#9=RiD4xqvWc8|JrrGep>3^OM2e+>y=?GR zVxH5*!4o?aKibF-`d?&XNGzl@Q|)|6UtdUT;b0>xK!1&<+kHlMnEfag(mL(*^?A<& zR+cO16K4657j=DHkG`_uXkrv>pZOwOVr(qY%4Wyfko9_gD&d3O$w!91R^3LfyfH%c z*$J9YHW^c|&-%J8nDrvBykxp{=sfaLwuFf7{s-~7<9Zp-XOtV+irm+hu`M*MT?+mE z&oZ5>!>eJ=P1IA}G!>K74wviispMGjFR^MJ7a`0SUtkLskyW_5i@iffEq&tJf#^c$=T4xYOhOq zIhM6_72$Q-(T783-NNPeF-5YQSy&jWvC?_yt)KUpEK4GEuHQU(&F~N{Oh2CK&5`@V zfGqaNg<<=dQeo##wU{)OUHe7(T4Fk#a|$Coe$D;4DI}~T3Cw=1@5xbduhJe*bj%*}ii$Yah<+zM`ec&|+NmAy$^%+E!kA z7%CV4?(AXjOYIy{mz>R3&mC=gZ26Hj(4DKD~Vmrh?htBAW(6eZTdER|qf0xejojH{Zts~Q273xqfEwtla zc%-}K)Rp1HCaJ#NVZ2A*Jvt@Wvd`M;x$NbJCfN9i?)ahm4SFtn*dS-qIacNdK zL1m`_W4?2C<-^{+FE!Pp$1lIAo@q;!_cwI$Rh~m=LTaXnwNDSM-zGMC=Wt^lHriaC$$aPhcj3M|I8G zweP8tZqX+)`6N2F0XL&ysK^BeJ4Yi z`RiyLiVe>_{YNxjx>xf`!4)hJI~2EXpw z!_6w}aM>u}c6IQn(en{yLuxUO%jN5>%l)xm7gm+-$3*acTzoUTT%Vh)HMI7)VeA3I zG}h)N?*R%xL*FWEWnbxzvO|Q zsBW2q*LmW`n#V_Z%L7OKp7l}PZAJPQzMr*w(BkkW(X`o=VB1JD#@l6eT6=1eOWZb_ zd8sdSQ0vRZ(G|L*=3-ye)OAKz7V;oNU(dzd4tb-h^soV{Vd@`@KyO?%I-TpgV~1FYJ6tVlv)?x4;YLHmAKJs7CftKYc1*me0l zaXImQORl(dT_(gWjAc@vUOjHQw1B%x*#EM`hw0i2JW_N!j+~eoyAjsRnQJU_#`#54 z4RkPQ|GO7(d#-C2AMLVo5>$s>8puF#8V(vMM_O#ECR(}HrRl`aQ7INHw7v}Ff372j zNQ7VUvJBX~6yVJg^~tG}evfqB4gR~B9+-Xm=Q|CdQW~*+4|Zm+D9*OVNZ(`g>)zE7 zm!%eb^OBK0rzXqMS$@ZmWmf)chjWBRkE-#VeRms-8H`kp zsIuwg7p6e^?M3p5Bg&l5()t^a@06eAzDbrkcerbny}#@!#>=k;NbgUgkBkH3Z@xz8 z*ost$ux&q_A!gv8w=|@GuIUl}or)kmBhmA;ih_~rdCg9IO$O|(aHl$YlUd24 zPDo_#jwNZrzxS{3ql4%!^-f8Bz@mRDPxzrhZ66)BEn_ax~!mc5&$-dK3)$qoHTXKR6H`!$; zBHxI~oQ4+PD-~~Fe}19xxMqt4!`(M3r<%QuL{GLIc~C#>GBiUAe|@Cuw*L62@OaGL z8l#IeOGmEbefPjzd)&p`yn=JVy3< zSA9j-L7uLJYh#Y*Su8tu?iCoNr!=Bh(LI@TO55#LusS(bAebP(#|rZ@=3(qoZ85vU zcZDCS1>j5Yu;5cUQIqG6?vS%6H5GV=3o`N0%({?qL846FKc}4i5yR#O6Yf_VR*~B7 z2jyO`b9pJ;UB?a_?0P z$2G9m!Jp1k!(*tkZt7|Mqzb;EPFg{yzYd@v?ie_uLqQY_O=KxBv(h=P%~$fl<|OFMyViKBj-1q^kjrK6zkp@aRg*N;&xt{8ZXXgZtO7;^4* zbK@Xke?E10TpQb9cc1jyix_Qdr8MnNiaXCx)iyj_?<&Ia#nw~|4RzL)4{a`uy+*(>qUCaL z;4vOO=UE@KZmb-A?b(!j*%MdG`)ljRXEgQNBgbiQ<8Aa`&Mzw&I&V99G&_t^=kX=~ z-00#J1-9%Q>4N>05mVMIl5=)irBj;;7>=_lx8-D<8XQMVJw7*=%FOYi59cQr!Vd7* zTB+fb;Nx7~5_`Y6XYJN3I_{n_b|LZBr_Kdwr-y5wOCQNE%F4amErXmwTBLyoxBUX;|+tv?~Gi4Mv$Of~>>z^sXojJ|=`Jh#NizvdAH%Nz^ny?zZKHthD32U8LZrQu$@J*-t=^sY ztz56hKQTEIjH$^TBIjk83ajJHh))$-Zld;-^z2^(OHk!`S?5#&Vc{BPSs{LM_0en8 zFc(#q{N`4yy+*k9O-@Ol=2oxN)aAHm&I08bD$5sDJ>F-&iuN&&3Z8mY^|T?VmtB`D z@|-6}*8Zu+{o&ZA%dGTn8w#_i6P9Ck*&PK_%y!hdfdQY?C0oz#NZaPSOEqUc| zp?Y#YdqToR~^woX^1U^^G@J_8yx}tWn}_h9mVU?BRupscUK%Di7WVbk2&u;n=S_ zXNQHok{_RGV~+{cizXUJo|B&vw2J$JX+y|F*p=A9y?moVE6*U)`YZNW%BEHNBl=xE+VhhNm#-zgN!@LjX_Qq|`?VxG=H}3WUaZQHb=?GQM>7qNuz-6S(K4^An00>H>}|vrx6Z9O{?O zI=oV&t@q->xT5cVkQ{66rS+ugr7fm)k9fk;OWR4+OY6yz#cZRj%`4>|LE}|kOVQD< z?Z2nsL^%UXu?uC=_31oFM?WcQe1mMFOx-Z*c$ckyMA@39UY)t}z6VugI>}~;(W_R8 zhbf(q+#+vXq4NF=%{IQ5&i<4`Ur7-yDGoi#dV3?Va7WFikW2aK3Jaa{3ctiHx|$GW zYnE4YEMDch1=JB#Ogqo(@V+nqGez4;#ttw;9%qsN?2y&wEoq~}_ZLd!`&3C2B-;be z$X~IvoY}3h`Gs2q|HxaAfyE=_4IlPCVwVF^DXp}%_|*u5fXH<<9l`Ka#&t+XTR37( zI)C0bOga^MB+GQ-Gq>aBq9WrccGedT;X_6c`u z{`NTU1c!LW&MIG9+6-E`si&xn%+g(ugOUfOfqi9*ObXI3xscfgN`pmH6~d)_`*e3p z6sf;96%uCXj^JlHR&Q9_X44y?FRar4R({I+!XuNDpnwY|Gw&nZBdyewV0jAF!wbC| z)1W)!@BVQ4!@`TH$K@Yc`WGz9W5wj;H;<~^>2mj;U95j{g0suDD$T`R*-O?tq_|?| z%-WipgaWpq!CeJil`IaRa`Ac zh%L}pnV`OIs{hJTfM%`V;*6y|_}EQR={&^hd{l+DVg~Y~WVEYAd&ad_Mx|&P^JHp# z_WM!&$5A#5&rw`m$pKt5Z*Cou{?u!E;{ji<6~=kLr@};_&y!*? z4`r|AM5ee@DnhA)--#F@pGjMz?q{D`}5Wcv^l!b?P$ZLV&pl_0nX0l`o%rx zdN~|dvtR}WpF+(#2($tsi)qc*| zRL1JVw}ZZp)_g~ONmj}eT5Xx)hU?iW*32X&q$Fu0^iWm`=TgNEN2a^ne%@`ojqHw0 z^(8hG^(7vmjy4+<^(E>Q^(9J_0k%Y{rHXJzo8HbM#;gZS1F2yZRB7+aJ9cWvUv@0Y zdhkfdrQ94Yy6w_E9+7mx^O3epv66NhgRU7XF8sV@9QzblwS!FF-J^Ok#g3ud_);!$ z-J4@UhrFluYHD7j4jXJt*^gb<5~O8BZ(=E2H!sKsV%L2wmq&-c_IB=%JVbJO6g-ub zET*(!_xnsax%ZON(EDsV@4eHqaAl!2QnqFp9wlS{riK9XehR9kir7=-1d;I`yt>d; zwN=`u9;3B8nXLq{QN_gl$@5Qp_iN4ZT!oqqOKT0IloVEjPc2W!CETwQU~H96Bf;z$|9}ugs3P(mu=JysDbHohu{%%Wb?Caxl*x1dxNKFi{;8W!7_>$l z8onM)ntkb`c!plGRs4>1fMYMkfs;e?r|iSU`kHIUO&oL`&mnY^inw0Qwmg1O@VH?R zHg~(jr_0;1e|$xg$*4x%Fa3+F;u2@w&0)$oyT-x6(ZgYOWGK&Ysao23t)< zji;=4F30YvM?Os%%_-B1FxYrVb>Ij~1ph|V#X@-Xh(m)<^mF34d49A;Gs~Jf&c@{A zqlRT$^v$T~nXY-BolxP+FRVyE<`g`tZi{Ng#NXvQ@ zxcZc`g@YZTBWTy;^y2u+WB83l2Zf*8}RWJq_HwK19(ReE*)F_N$bn zhfHHK?{h>G3qqM zW7H1uuvbyNby%t_U)V^0Z(TX9xtMkaIjh($(p$$Gz_nwm(D*vui9zP{t0>9PS5b=E z3r)eHucG#azKTktsAsIB4B*lwp?*$dZpq{kt8@AC5x*0lYgYaV8EbRPcWiPe3M6)K zE16a-OaFf2*iQ!YtIzKVKvJ^LG}&uMz=dKpJo=xVh5!j;Bg z{(lzYhDxyvib;gD=y$p>OiMTg9l?P!;FluqWKT5ytmm{vF6-A@5WbE`v^YgSc zq(7XYlmT3Zn(niMh6Bk#gU9yyidD6kn%N?}J~FO%2(Wi0J}sm6(Rs}#J$-%WN9lss z`itD`n@#hqX@^|)?;9{^>N4cae{~CpS|Hr$5K*{3l7GiK{QZYRqb5mg?@H<3f4<3n zro>oda7oT8Ao#F@-3jZK-se)j-LWS%dfoM}4dm1eM3p9?2aGZ#3zY<~OB8a})RQzm;Y6R=v5<+02{B-Ql0>S3k+F@#~1@Q`GKx2>y>K3%?DTE6S~ zTbCReK63Mp+}^D%dhMg(r_DaCV7&vlh=PJAoKqL5G;mo)W{g=zJB$1cgvDwYMigd? z5Sxr2KVXw%y7Shf7mO8cRo5o?2^Yg}Jb2is4bQSHZ9@o)MYY(A;f3Vfd}*3^Bwd>% zB|YW zoavKZ9l+y1(f4I&+cWrv@x4;VMy53TMcgcqMzJ5G4AhBYx0C4Gi}fGV z6#7a|6tvhs&h3!xNfyu3<+~jl>+if*MudTL&ojiC^4nE?oAyX=pP`wNB%xb-bc`>a z6|H$0SCV4zINi^e6DmA*f(7$hDfE@uHCqGAMdahuhrL0kQAG~#CbjKe3+GzBwu8>^PW;5(PM#X{SQ3Ad1kKT)C zyCX49^|(=%+0m_kw+eB#f)CC-Jfy=LnA`ZyUtHed-thvelmWW}fv1hRc{km-eG&3* z?`*jIV`7n?)2qCLMP^kOf%th5in1)uP{{(idz?$fcp4FvPEnOR%a?*#><17Dap5hI z4!iK>n8McneFD!+8sGT5H{q$i7<0Hu{(-MNr{%$0bvpS+=QK`n)85x3!nh}DMk2Bc zN8+g_F0Ea9aQ$3JOxTSj>d5$ZZ}+4MJ?JIA^^W9ddOESXD+x7;0w;4PcTgzen$5v* zZ+k(Dat|ezJBRa5aC3sEQ-lp@{BJ}RkFEI(TMAB<+)KZAFU_tdmLRQn19x!g4ggyg0b=CYM(>%%gYAC z{Q1-CuP$9lNDE3AS}wL>aaRyi5jk}$%f9XE9`_^lul#D!Jba8tJ7R9}0Ucgug~v|gm|9!TmTLd;5f)L{ ziRbw8CFTvv_;-(nYE_P%AT~bb&x{-!?irSg6g3lw8F2J4aH*e}I}Kr%y{jmam&X&| z_j%Wm)O63dm-|Xs%_?)u$L$2o+f8y>uqpj!CxE^u~yxdjmT9Zp(n!PLSl#3>t z%ek4(FwuVfPwnL?F(tfrB7*3E_`4Z=Q%mt}t_&^Q8GYe7G?wM&FX29nFEyKKS`Xav zdUAn}A2a9E6JG50R|E>s{LIbWEEU1))69*EEyQLyS5o%q%^gZU|l( zF~C$Pu8ib=%J)AgslIH#&TkqOX52)?GNgz|SR7XxT3u+UJH^-bNhsxra0ye{u;0-hD~GQ?;5h%ZOOj-J|mioVm`qF%or2?Ro04?b9Mpb6Pv6m&!w_hoP3uuJX0A zCIhGkEi&nrDyx0k&{I}7dZn9%MBElqUga>lH}_z`WPd0AMeU++S9@2%*Ds&Y+CIB+ zLrLa<;>So|K8<^J>B%YaU(Hwcr`Ydt#3_qK&9ShTX{9={bX8jh&Nfn`H=QYJHX-*o zDj8ThIjGT_jz#z8He_iUB!)|7} z&jkKiXGKhQn9BlJIt`w)-1PJ}O|}y=A-XBOh=Y!qmZnK~iM-fO4Ls-p=KZ0GEUC2M zjy_CHo!^&?dVcquk$}&WhudlcO#ECdcBKp-jk`DE3=e5e%SxKVCZl*kd%lYS2avB*u)FGr0{FqOiW zBEXQdvjL%rm-kGYMv9dr#@L-TR-XQtexaA9PvlM`gE*W&7VCCV{H?X-iziAu#o6{n zOe;ct`n7?Q)d9NMK--woTE_;5&Y>C0z?_qg4fPPDpZ{9(r2xV?s0<^F*KtJ>AI2wB zGkrdHa%E|JXo-sdqPGV7u^4X=Luy9V0J#UMUzH`_XK;&2eKP6!SU4#&KVux=zN_p( z3Ly{usk8VNrjM`dRAF58T9?_&AZH2H46DatKGh~IJCk{52m7j0G3H$ICvJ#RM$H6L zLZ0Yl2hfdLQf5L{txi*%+E1CJogE-<)ktYx(6E3Mw{oJ|D*RB+=*O{(Td_pVkn9Mp zbZ*Vt9Gu;aEigv)r$tWQ+E zKSXs(_NNf7Kwc{W5NcEGj#&~%af449rb&(HfI>C z=x|a*hiZ{x^Nv`m<6^dRyZ0ZiKgbWaT$s5w=h#}!nt8GBSxIjD^c%ap5XadJLcnJq zG5qJ$Lp}X3#^dPsIF%(fn1$R{W1WiahDb@8oqx*f5Z@xBUMw_qxcH=q8v~o#b@Po5 z@QJ@6*D)R2?Diut$JAx8BeC}4jTttUrPyHR}Yw)ETyZ@H_UDEq&YrcPb|IX7&sW=f?b|@^QL(A zbTCB+r8r&IUImJz2xdR?4xLdq{k+VWa)TR>c*U>L$Z_pQG$S7qbo>m(cm!4RZG-A% z+HSA-O5n|+jB8jrUg$G)$T#CU{2VVvnaPYO-Icm}taROZ?@OD_vg))}F(u^;Sx4nK z-x<>@Un2dpcMi37&{d_;l<&Kms+#&5E(Y{+%mvhobl6-RuJ2YCdV0hVy_apI_w3MFszXZw(N!PuRC6=?diVSE4tV3oQ+QFi?kEcNp9O$dL9ZrPv52VXffuN zn&w@Nn{;-9j}l^1r?Kk--9Ge8BM)9Ok16S;){oY3hR;QMc}0pg#*Dp~@h3*A-Y;aB z_LB9+mjwwHMfcqBUQGKSs=PD@)ofwr4ADrL9}&zSe8az)mF5tvsJZff;9kV}M>o~A z25pX+W3GL?HRiA6JcV*(F^3$rWHnL>(ahv)uu&Ii1ga*e%XPhsJt{6xaVcZ(_f0x9lCb5zZ9iyd}w-Y zs%edb=ID%rmML-@&`m}c)*xPBb}4G}M0TlEQXmtB_@~=D3Mi@>Sc-RWCk4LCgLG(U z3eqyZ*|+WNvWMWx)-3fyLY0@ieyK}WS4f6dSM=IEU03){Z5_qSj)1>kmj?Czc3ql; zUEt@swB3s~x{1M#hMU93gxforI>#g!y!hho0GJbd^d3e9(7lnWikmp}{&c01fWf78 zkCTn}zbJvet)FCe@Gq&aq|7Sc@Rd9M3B|ll8PGJJ)%P%5G|}58)=Mqu)D5TL$Ph0JsE)*NTKXlv+QFl_Xx6*~Cw=#z=3)y<33`rwhsQkizLbfI~TWeOa(qB(R zxU;cu{XlqD{~B#rt8>lmr)^gjv8ioYdzYR@H;QyF5I?x2Xr0?S58;g7Iu9B0IvXg| zd-JAQF5FaSDQi`DQG8IR=g5-yBlU*~LPZ+5)DJu^J_(P2pjyLpwl{SR+7OdBi)Uly zn@t<%?p)<_RuDP%Ms;%ebe{*m$&-B>J?&1|Y z-Umf2@SYkOT}L0=!EBI~-A1deccI~WcDC^^b=Yirg?PUy82FOnYBGAKYZH=~y(HL)s?TzHdHEdd=5zxmRXQ zPw&DnbyyNv?&3R`4elP*)4SlfMJ!`hcFfYC|yT6msYU3P(w+QW|pAHX(v)SONljBvV%trN~xr0+$>5dH=A?t7T2wzvye0zK> z)Rf(5E*;h&be%x8&p+ujmQUvK7QR&BEXH_@UvEIror`L$B(XBJRncGN$x!pKPnY+L z6P)d=hKZxY=Y4x7Me}D?qo=&XA3++rckIy>wx1*(@S|#)|v%TTFcU?=d64sk%)*IPStDI_oVe7jab8?XOlN|iE zsLBmHT0l*7BEa~Hw!MX%*@@svzhK0j#B*W}*y=0^9YU;ynqo&@Ay``N?JQ$u{qjL=V%MBB=vHpTw>}huLo}3}DbUx$$;S zdN8Ldw2=+wz8VxV0}j!(vz`>vJFsk#l(*^c8Bh?PWzVY(I5^Q2)H#wPu1HGKU+rPy zmQnk(>2DI~fD;6}L1o+t^qSnuETNNySMOsmGFhBcDSq(VH6QRg z!k-}GVOO`uBr%02x!IE4-ZA~nTQ@fS^h{OF7bTV@y#@Ub&#>){T6iQ*U55Jb-i1c& z)2YA=ElDI)K^@Ewf% znMi*j9Ts}F9^$6;JI#}8$#q(KD$?K3itYS*o@TU_GD+LkgXM*J@}3S07OJGYl^scW zrz-~1D8NF`(29Xs3b1gjmfN0-S9A}WHS*q#30g{fuGu_DS)7I-E#tG+?IY2#+7rsw zEE~1lNqOjBTu;S-WN5{JdGgD3g`~WB8gWx*u&|Dz!y@o+yB@IcZ@V5c_Fr6&?MxdT z{)^*C{wM6J56t7s)k*7w*!`KTWiop_cCFwVSEesT)>YJAm3vXq3twtZ-!-+$Iwpmu zzd3oz#3oOe@9HCo3Dsn=nB9vXxgw5ER+vYB2L1BR^>9S$L zB5319kGQCo_s~6YPWqLDLml;js#kD=<%dVE-hsOV>i6(jEE+Y2957sfKtJz!&bng5 z`D~%~V7sItkG$Qh2^$V}Rq!fqFU-va{%Sq{6x>21X2Y^!IM?v_8ohHqo3-Xs4?jQF zXzNw_~-8UAeE!$k+!fGIGbtmAORE z5cQ?R0c)xt(&OI5=-h|L-md=S@hI@V~mGISa_Sxl$<0vqj{&xl!=B zIScgs--?_hT8IIB<6h`NWlp#+d@rW#fhr@mfq**OD&LqR)ASUy+oy zJKG0PbaAbux9`LHYoGn&8fDyU`w;Ob?#NGyto@Fo=k7Yb@}Mu3n5mNM<}h0%0U=Ha_rV}mlJ z3S|26HknBeB4|Bl^!i&T+*uC96=&nNw9%X?wx8`5oo5*=)Cx`tAvxdb4_6Dx-If=rW!^~Uz)$*gef!L z)caLPM$WeEiLdXNIk0Orphibx*ROhkYd^;^2|2SJHB$U9_JeX_ zR9X0lGZX+Tw*}))(FR$&RynfFcM&|s_z1bcF9BPSxA5~8ddhgY6Vu`1{GWnFLd2du1c0ERL@$j z{01ib{|z;_vf6&L?46y4%`L3hGnMCpOOs2zs`>_A!oL&LcCMCGa11v?@b2U1N7s_2 z#E&;1YC08oh;)OmDkqz!t)cg{C&V>k^m>V+i(lxF`GL&-y8D}ZB=wkB!KsA^Y~MIy z%$vrvS3>MdeLs4lxW;LCJi3b!zdjsW;g{+q7-uT#d8C%rhQ+|)2rcDVuPt`+thYP-q-?X(nU%~EAx5h=M|hHga(^W1xGeWsJBfG-GNK0%>1&oS zbc>SoF>Js0s}}nG8_5WH;njDhv4w5*ebesu?{e6#3{bC2>5|0X(Dhm@A4Tl2sYDac=YZFd_f~+?sgcp7N^MN-G5PBdP7k(Eg zx_R}5%3Xqu&BbC^)vvyVzdm~1P{eS|;$iB=KGS@#iINT(WRxQc(poi7{CcU_*mh@{%)o)lI1E+P+{`75G%{XXTb`NZHEVwG@ZIW5h7G2T2++ z@u7K5#;-$n#@r{nWI~wCL&$n7&=I{vAV3S-H(&(=zN8;z)uhLK8D=6NLT!ayO!?aW3Xfm_uLvc z@3E&?S{`=@cAP60^*w7BgpW(=Bwf}q`A+TI0*Gs`$82!G*ZIvE8y6Sn|8ksm@@<&$ zP1yS?^8Ni+)9~}=^?}jv;UlWn@8$SDZB|fowwue(&*uZzr7H^bEguc-VHb}OT_MS7 zkDK`3h3D?Y1pMin5Y|V>Mh0&;CZ*}V3r^pZ?$G-fQSD~*NO_H`;#HYLA7ipaLnV^3@h8zt z-C)UZ4JLGbHHBwoUg;q(2os>CeQm8V%rpGH+Q^%BXWZ;%Bh)Z)J3{u$K+k3f>!Gc$8ftSB)ptt@SUm1~;k+lWVuC08 zdVX5`cBrDy75o6j*GmLq!cI$h-Cf;2?>8&9L%&h-d^nfdYw;UaeiJ-0j{-(uDKp8b zY$T+z)9i{GIe5&%UFnyQV~+3vChBfM1cK~0!Zb<|&Hh7T@7{npzU6|$g1?oCs`_3= z$7vfbCF!eJC!i+Ay!x>zMO@un-(L@vE48b^rPJv(e<#IwvsVkcwaLk>hW0x2s}U7; zR0K9d!wV*Iugybny@RALd>}WFRLwOfll4_S*VRyM*p`?(7N`XZP>=Lb7@p9k;ofd> z+^@;U>&S4oQQ5p{>YkYKt=;Wnmot`-?kX(z7URNaBraHKav)^2ay21SCQ0d|&x#VG zqY9Jhq}<5PY=NFErHNz>2S+ezE=AIekD%D6=cV?o5y42+euMe*e}4XiOjcq_?eSdZ zS-gmF8`(Rqth`*7(2S}Vc&8v!>ZQ_<&8LXQQZ=@-*#4ot?=qZ&^S8oAN}PSX9D^Z2 z>R&OvvHm-O{EuJJGw_dSr)1s{kn8VYmaR)t^0sSeZ0w0oVVqN}rMQpPY0?BPIegXr z1JY&x(awMR<-a%aj_@Zf?@Q=TZ<3iL72;LCwC0GiPoXj_B$u(3feVAt3< zH$5-HSYUUehhtgKxRPPR7If>@iWN*OpWeXaOc!&BxyX%pMWOJmm79pmD1w z4{mt2^rZ0sY^i?djG2C1p^(!t(n`C;9pTBRMq2zjO+T*6%|Xb~4R%6_Jun$97Dodq`s$%rv!5J|tE*VF!~w!t46JFnU;UCxK{79XRv zj_bklD>Aa;1Q#6mJ7{@#g%EqFLaj@#jtEEO_92-=E6tjRHJ`qVC)WJjva@Vb@@i&`{LUelKvrM{(nOz-`58>NJrgT%qCS&HM zjd^TPLV<*>f9=cSmYM_i&g2FH8@w;klvBAXTPbrf2AOX8`OLtV0dy;VgvfX#O!Vn6 zvDakL+ivBH8kP7#H0ph-^ye(|TjSC@3)t~xcm=sGT`2ZTs;r~Vk49nnpNk^^6Bhy< z0YYt0LqZ{%>RJ4X76s!&rbL%7LV|o6Sj<1O`az3$|JFHb-Dp<`i7ZW?0uP> z(pZT@;N&k6m1JvIPAwuLe9{xGL~MB_h1cl(g|zx+31vA<>n1zbz&7ip_f9PCNM6u{ zRw@&Ta?IZjavTHTbAea%MstBe&|vs5F8K-|_%JR+bAc#lLSLhz64-$r?*|wo!I;N# z@BOdXtKAk)#oLIC)$SU}2hL_@f12H<*KN&Wqp*c%C1>N}d^=qyJYtJy*#3Tq&rNYS zt8@^XhQ;6rjQ5XD$sv2Timq(-`o58XzeKWV@907UbONKyxyoo2b|G*v_rxsfPsK>+ zbZ^WvlXx^N*UBThkoR1aYcT2uMzEG*aiCTB1J+|lRdP`19>t?!%c(8e3zD%G>dnMC z92N2u6a!!O>4XQ$SNRiqLxEwK;Z{5zbW4UtCg#)2>!2DDqmOoh?4XA5Wi z^B0B$-W65St$X}*_!Eufh1s*5w`>ZfL+0JrQVePdmw3;&kCl?4Lg4CLZK@_%*xxBE z8>LN^nxM}AItinr7u#Noprf& zCX^2zh{X1IG&mP1Af-nN#QPSD#QP3cM<_&i!FWav$iYugP zJY*|5FKFDr8+#X9gL1217{hm8i|9OBy*3Y^4ohX%)|@Fl%IzBjEn8`b>Pe3U{3XKU zMs&vV9n=a2XaX#{3BRf77h+M{wSQ#3&*3uIsT2ljOrPmQ_1EsY-%mVvvA_84itE-Z z^w}Sx%xm2B(hQ`Pgj(5cFQ=^xZnP`udhquT)Ol5>U&^Y9r_5MjC5c4f;fA=r?66-wV+7cWrWLV3~- zE22KQhhTNRMymOy@p2#7$Z+<;U(F|zK-Kz3@VA|hlI6QE{%w>9Vg9y7ks+MjDlwbe zkq#qt_8V%=wv3 z01&OAEC#9aZWYL?KdN&o&{4u%Gdx_$)YysW?o1JVVp3&JGnb^>;N#RblNQ7*z|EMO z9)AFZT;a;K{df}bwV8Amyju)5THd1TX%S4=YuGrryz-h#cUZXPpi|3>sB?>CwiLQa z)OzvMp;+na^Mf!P(L%~c7ETO)+=ZMZbAtodsPZT7VCHhAZPZ9~{bq$rXE6hxL=^D& zH)SV5iBi_8{bd4mk~wa7nWA+DdWoFX=EI5W;#!ZeRz7aED&VYK;_Hf{p}N}fpm^8=9Wcy;}IfF%Do_}UZ#41rV2N#{<^ioo`zGWr%N8zGe z9aC;+C~djyQF&uU)J>~W*gW-;3Ji%|{2a}kHuor^VcBuEQIsN_EzW53IX@##4IUf0 zmPU3y_RR(Eny^LN0v7ek;ZibI-b`71LUd;2`=1L}gua@WnHqSCO*~$h;+*AQvG_pO zpUMFyw?GpIyt;ShYebj6*3Nz6hwBd59R_g95nAg8S^TfM;))04r{=Oy(dM4gVou4u=!kjdKYL=1OaK5Eh2CaZ>BYk_Pl(*|d0nxMrO%Q-z{tEGY;%Nb>&gnes* zzvlWlbQ|9sxAvWSgqJDS2HhL{y>9!%a!Yh!#qBT0lz_HuM2jYi#;5Yw$L$HQbwoDg`o5U4^1gy< zDNWnxRvv;xJ$uW;^SYhoIhUZ5<4xldQvfm+_6*6mr3;0&3fbUuMBk-Ol3~fNX1E&v z{73gfX_avPTcwWbb{bW5zvR`U(y=WDvqk@aag>akJ8!LNYijrR7~DD}WveJVFSye5 zj7VqZHcj6NwGC&0qxu~~Qx*Q3kwU0tvJT&wlx>Q%j(KIJR%q9h2TaK`f#KP>CWbLO zw;ciDC%LbojwO=)ZTCJMYw;x%6HjSSyO@-_yUE3rTbHfG8KhSQgk4Khz0K$+l=f`N zBgaiAq+{k;G=o$<+P>m&qfzl-#gNg(WQjFPc&Dr#H-Z`f3U#$YEFkmF5HaIrRJaNh zE-1H1M?8K|6P6PrIM6~=|6KW{<=N4~M)0M%rM>b=c#`Obh71c83H}Ar){vx3unUqs zs3jWv8LBMEIO7}O-|}1(0J5RXW%csAD1a&Pa|4T5MUguH zEbt{P?jk@5VkU47Rc!>;^N*3*NCeWOE#XmwUTrso`!mYgx~>u>SH1}tdw=d#-%o{K zdOy97>5Vo0GtTEZ!(C`R0vP=m&(FN2#p7f|SFvINW=k=@iMW6G=$0{e8Wa(G zp5|~L8%+6fp9cRuGN|JJdlKA!k}sg1zdN;91gF$>W;djk4Cv9t-g7d1FR@( z@ehg(q6tB3V|rp5^Rl>jy_fhlQdnLhQ2^}Bp*xYSjCnz^Kbp@A`-&<(c&uEXeoTz_4}+$c?o!mY;K3SD;P&*Iw4c9_bC$@$ zHSQLhyN_a87Qx1S&^$&;O4Btj3D5T9&yVo;-KNyvjj%Hy3#nnuC9pFf3gqtju(74E z3cD4;l-XY?K?+=Ruoh9jNzQ<*5K2B(;Q6mVFbIPWOq6`+5b|HOkcIA#o&g``AJqdCxaO3NAm?CDnHy}*fDg=$wr4q};3en+Ufm2IbdN~Fi3z+5hZfLt!6y&88Vr9C9LE#bgUg8(r1 zeh|o;0T8$?Pl7NocU%z2oBTd~K_SKLu+2Hp4sN~ zp<6)y`=2WyUigR1{{cood_R-WSl%VPMni`lE~Z;4DoWgQhIc%Qh3H%oe6P zRu*OFqXkx|JnO#3oeH)!4|RWQOUs7 zuhGLVAZnY8C0py{qmS?ah}7Dw+=EYEO7^kV(KEIlL6Of#Zh3YpYW(^_5)e%@p+ZK=>(jdDnpORWXn&AcxB#lU3aWy5h`p&X|-c8|G) zs)weY8ppZRxOU8Vw$0R%yvTP$Kc%zl6!u>KTj%YvcE29Abc=%M80eLqyKg^(cv>tu zi;FfbY|K66*`Ct5ph?OyB;0=Ph9`l&pNNo&*SpEQHs5w4fQKiUXLG+JZZjae#3sPb zcWh9)w?b|XwjBVfp^@*0nw`IS35aVxq!5I>N+&sw$sImb!njMp7Z46zV`dX zIrWL?+UYlB$EX(*PpNj5DBaOQmE@JZP2&)E3kPP65WkIT(uGvd#{Lg~CN*=@~ z3IRLOU~9+<^;?$!>tax~N&}E(xIk-6S;8`ne4T6M<_fu)M#)AVkEd_Lt`0u`hl}f{ z^UEYZwiqH~PF6V`#YQXH(FM6B2_YoZ6rNz+{KO<+QIS;m4C?hC$XVD9>8gtKUnCBC zB*#xQ!i!C4t7Z|F`&;wbrV#KXBhU#I+jJ)A*^tc@oVhOgKg4>0GEvU+HizrfBU(duY+K3d%A8GW5_EFt zLw)YBPF=}l++9~o_YoIUvP$UKvg92x9GzU65`|4BVpUSZ3xSxh)WuvXdfvk0+w=H` ztxf4MOZ_5$?|sQLdJG3iL{8XZ8xVaFy0%p$dh2!#;)l*3&rVh^RF$`?*)jf?VKZPZlP>N9079g<;3E>|P z(dh&Hd{j!sA@VWQwOJSyVNP*Vafc4^JX|pLuWTL0C`PX zBe@d~Dy&5|@tvD8{ErccGExB_h)^=J2ssQgue1%{Vt1QK?)euOJr zdoFfGmrdjTzw4ZYfoT5v4=LdFaM3Mz;!xd+^BRV&!oK={#S91BDTD!0h;;*WQ8X~p z`U3&7SDJdpDQ!{WWvwfL8EWVg3TPw{Ii^FXzWv%YEw1EqytwIWDf=j7FwayFEf^rE z(Pp@)w*n@ASxMpqlN6~XI|Ep%T0XM%p5Z9io~c)X$$J#=u*JWq8-<3RJsaDfJ~{6wbA~ zQ&L!`#);{(;SO@B^Vss+o?V2a-5!QnK*uA7x8zGG85<~MBnTeC&32>2wm661;BIKS z#u3X08YE4bLQ4xgukqwk*$_2mHwKkDKwJe};LU`GNN+M|NCBAxgbIBjDFStq-F(`Q zV7L&elec*BXgBha)`n`OCf|@}t!0vq^7GFQHl9io+^jV!6%8 zqHT22%ACZUJ%zUnhR}D%;l-%1w4x;zc3mgUSskIWqOs|MRBsLx-z`TL#0`+@W~K<)jkweE$(Aoi2)Dxf#W|hwb(o z(a+79@gI*`0h5kuwS5d*C*&5lX}kJ8vG=gL?M6Gx(5wMy-{K8&uMCQ|@lKy_74Kd_ zlg3*qVc3#4&uMVC$rM^>?%1LXF|&HBtA;2FrISqD^e~1fv5A-yAaP(^@tvS}V|(i! zz+A*%M8V+e(0f8q9PqYoFnxh|LgT?MMCb&UTz*!n8aR<{rej!!%rYJm0~}fP12vM~uik#5B-3Mmx z?NtsTdiT;KtA)tQ%Wa{@wteL;1mBR%y@iUBlg>i_jW+Jq!jr$h1#13cl=I^?N)-Iz zp}|)-WluL`FOnXepugXkri2N%9SAaFb}LgZwm+zvv+%b)%zlDV*ZeGcn5WU4_HO74 zJJR>h8eRfnaYSNNdV*11?8CWm%q60Mo6225KhjPY+ zt))_2K1sC#2mlqibSSr#4;Vb>qndA)4BO~Q7P?3%+jWIpJz33ClPBza*HFY#HLoi` zO!doM{q_)LpP1%2yDXw(P_~}q2sAggn5>IS_}jzl<6*JXpO@hC>$cao#pl`e^Fjac zwCVn#>)v_iRh!M9G1vF=s`1^HE@A)3&1T)Lcn7L~1)6BBC5scqwz~F8xGq8u=?LEC z2<9x?TbJtlV){w^@L-t1dM5Ak4-rc!s3naH=!VRkaVUd9UT*L9@Qv!D9)!9Fe$nYu zr(j<1GIJur$c@Pk@!GbF5xk13?s+Qff*OJf77(GBP`;!p2rjoi4?TKUJmba&)KDaS@+p3$IGv$|b81Z6OiV#-|!=UL2QE~1bm_9^Q=w4NRL9KnU}0hlLYS5LZF z9@*d&;q_)Pd8nvnGT&s83nxaF&9oer@wYA3PorDrXXwN=*^FB*JJrh1>fa|Br*(1; z2OenWaLKQ_j?YbFSOjgA5o%MTRuZIf0S!P2+hhvGLWj`V()6 zLeY6!{g-Bi8Dwf1%~>x^BE6NO&ng#QkGdr)qCO8LBcEpvyMjS>_I%_9c3S4T_B-F@r$?60@ z4Kn^bnby+oK9ty35ov2XckX*%Rqx)-GFV?hu@flf@w5+;6~VZGt`}vbVP>qP+k9S1 zdHxJ@y2)lfG3jnzF3F&mz{b=mV)x%YUc5*=?l!0q^=28uv#_4P0o#Vs;)DJhM7{is zsc?eGnq(cuSx&oZAWfG-<0D=)j~Y`5AEgj>8)TM(uM}xNA><3-98dI)4`?uY6a{nlJ&`J7)t%!?|Oc2EzNxX zhTyy5zA9;bMm=i2wRzz(*^hOLVVi!*;lfR=^3r(HYO{Eld8a=bAArvpRHN2Gm7sxd zpLxmhR=d)BMRESR6&JF@{pojH($m?;wg_mcBw@!=fgZuaKO~=N#I?htwV88rd2+}q zs4f9I@)TaI7RC#+wn=d++TLRIp1+1$RgSzaTtBWz+2g$HOlRsq z>)=LqC+oIK&K8}Szp26KOhw~WZ=dTZ(SWzhHkc$Y2Ay!>>Tqs5=V1=n0{-3H`|B}T zu9PgnMVJTowKO}Wm<3fX({J&VJ~t*$?2kLEk>7HM-vW29)o+FP5|d^-4H)@ne}Dx# z6{Qxg4OEJs;cw~{oRkfn;fwZx(T!&2RiXtYL1OAm^l3{~9(@|Vr?={73waknv#Rrx z(A4&EcxL+=Vor^@VP240b(u;^U8XbLc-11!ENmVs`BYLnDx&moJA>az88e>TKA-mP zS=(76w#G*^*Z2_0yXVtz^VJ7hR8xZ#(5yRPgI>qD;q#7`?&ns zQ-tSg%phTTqrX%tXFr-qkA`;6wr*L^#w)c-#NEy2PsQeT;n?qx*gZFjtXH4H^x1&| z1?W;AjpHJw22q=a49Qq#EUBhRUBTeZyI9GM14Z%Kip>ryKGlN25P0YH zvYt?vwZ~ivS6O47i0|Lg#YRn|bA47`7n6*vWoIy?Lod_k+kJv4<>^cwWqh-W zVnQy(tqc0Z#xusP*O5>uS*I4GVT@wmCx$Qdss7}`QOCQ5jxl>_*XsJB(oId%*SBFx zO4LNF(A}3gYQ#nNx+T}~jf(ROJiA0+*ttDY*G2|mHKNw!6%50~4P~2Z=Ur|rW)Sxk zUSAIUw1VY*ti52hXS(pA728QFQYol75%7}x+>f?@4lwBM?)kVJ?tExnHL#zH+887k zX%qfdxHv}hv!9D z_>_oV+!HXXgHWf1-AKMICu#*uQBxHdK_JGwLCreMSiP0gf>a>LdCYj?IDmU-8ZYF zzpnbtJW^WgQ@%6NZAEz8v$B_bQkEr#VJtmCtR?fImW8mpOu$rqcx>TF>3a^I(DQT9 z9L;2J?nV{qg_Jt5%w3U>kVlJ|Bq_jTR&9I%{j?ileU&>wYaQ3b6$NSYdnvF))J64M z#bPn47a#9qwYB!^Z8SKzcNchdzxG{(MwpS0 zI8z+`75|(~EItJYd*IHx7gJV?;)xnMhSSFu`8Ljw%Ab&Nmc#Sxet}KKr>2ZS(B5#R zs8|#cOMBF?B@Jq5-Hspp4Vd2sN|ms=4%st9Ua)h1@mx=p%tM{GBkA4Vs96-zPfgtz zwK3~i;dDh3Msv37WjowV7hY^ix;AvH z1agwH?~i}KQ&OLjX_oX!NsZh`FIwJf7Z}=);gwrV;kQO~-jk2eFB1RA<@vi!dN9byj>yq*Kf zbhMCW>R6#bcO!>f6Vvn2;eBfF(Q_L|XboC+88$00%`>-hj_mvzCz$Q)_&ol3*x1;( zxOn^h=cE!@v)2E}BCiK?;>W8i0}pyJe=E~_6CXCSp$!0nOjK!F(&Ar7hnHaj*zCHLRT zzxcNdm%x)C+YOlqo*;vJC)^I~id58!WI8U_0QlG90-4Tz(OdRzY)T!@s-NWGe0T1l z=A3Ep0mMC6tx{Qu4dO5CQMuUP(*)O}PP>_QmN{5~0D!$>Z8pw6k(;ge-8b}`T{k5O z_0rY$M-F|9{~_U+q?4Ql=Yrq#`Bevh!htFAyl82?_v53p>tHHh5Ic!Ck@oyL`oJD{ zS-&1DsD_w?WYR$!RG;U3-i4=9<)1xEnB;*{*04)r{k<{lpl8Mr?5j|ZuC0_fpEG2l zkU#rvjKy%er8$kLUMp^W&Oc`b14nhr&!ZEn?ijUb!y>V+*0||J4dW5`O z4N%Kn?t~JTgxS;>SrC4qh+8?O=m+b-m~-_uAaOtP_12qFcsQ+U4NHATx&2oL zLnjwbD`Uh4%`^u?G-HM4FJYV#Yif*ip-%b>?_i$f%zT`qEz4%Lw+$+t_3==Jnk`y$ zT#Tj)E$LVDL774=bzR-EP>I6Ypg>v`HiTA?lPe+{nu$nuQ)y4L<t`?Hs_wGEmgRJQA_>Bt5}WDd z7>g?aO`_fYkoTN|CDCK6ID$&fhgzCyZ@2v-cHdsz4Kk8TpX2M4}@ua#IpA^=EvqX|#yd>A{yjdYV+w-!V(Af)jhH zZF1rLc_UrLLQN7^XBcN)lO9`yQ7<}Zy*kgFm#Lk{W++Pk3}$C>x7Rh=opBT+yAn3a zvP4vjNb6KI4<88K^i5%a+2O4*^nd;Jgt^FTJFMhH1+HYH-IzRm68}NY2>oF1P(0w$ z_pAYs04z70%FURAH`9$DW0q8w#AE7ebI>d(p}e?16(9qxXa#BRGJml34_92mdcKDT9I zVkfkhp-n15cCv@@-&Pbo=%>CeX`J%^WJLwC{!tKIn6SMBzdh9}NmP#h3FfSXX)+IS zVd|hVjoZnvxIq5lIFg5ezl@Dgr3~XGf@MpMD}jZOI+n$WMBPoJA*HMU7^u1D&uf_^ z)RAS<^a}|*u_b`80pop}^`G6b^_-&hZ(X0qw;nt%=t&)(7E>>k?ct#$G%z5BY(#U| z^*#GJH_jb&0lCZdKDWBkVa_&>t6@Xf)s{azjc+GYr=vU#=sGBm7Y(AO#)0}_M~uoF z7ACR*fkBm>FFpKBEV`P=sUe`NVi4%rgqh9((?=qHJq`TL{03NkQP|;5TQ0juE0=Jw zIL;H_6E+A>z;+(;LP;cI+TQKZWsrrs480eXqc$p*KE1N-nJ;g6vSL>0gm! zR(&nM>RqVgumjXJVBR@ou24ypzkS!Kw&3ZFRK1P`i71{PJWPCGD-sjlzoNr6y zUU<#_HWkI&wIIv=k*?sTc<x|ubh#XQ6Qe)e?IP(7HcD1Y!g)4Z)UfZ|Jr8x?79OTnB1 z84jB=^MMJ?Bt3b$){zLf(6!WW;)#GwsRvNsrn%JAN2c1pidK{`-F%6Ar#Ot-=oVjs_Fd|RLiugn0`pg zSLM&-UEwU&MMt~C2ERjgxos*)BrPN|Zjahjwrn4?Crw~p?;YRFFni0nd>Z!EbyQdS z_U`BU2N$5{tkZ3bxouFIW58Q}^t`28mi9c~Z-di|nbSSj@svG3Z&0>J({FI1@l2MX zU5sqZLI$lFi~5JpmANO?U5tSp{U{l}mB1p9Xfz+UB6g0^h5-aTOeL&- zK+71Jbx@0BvgIPnc#bd0AZTvjcPPF8ksp`9vs|>l({gPqPMhI3uQgr1#+ahv8mIMfvVIXDXa%4&hYdx`(`F zx?ly7#v6Qf|RRrg@4g% z?_?q|5z-Gv*G4ljL+zEBX*PJa6`c+3%r@KnX1W8LDgl#~4^s^c^N4HM2+8dRyX_x2 zjvi8TfHWs{Nbc85iJd=a)wTApzmtuUsg5>Yx`d>d(QU(JhqiTpdz=G!jgS0+1{D@2h|_P!i~^)xruB>9b1ALBPH?Fo zZ}*?XoA?3e&azHQT;HXd$%(BxQ*JyuL4pC&5n^4T#wa;t(Gx1*l&bBS@Wlav6OXCndl*ig920AW}`w3M#Fk8ILDz zE6Kw7wb$_lhkkM-JY9gKQCHIaDIaA5pw5IEWF=%Bz2HkK%M1PsJ8&Aq!M+%`Vm%UI zI+%$`%Dz0DX?vE6(O|W~LsF5Epe%oSB)c9T!+UySiS2mkwT8cee|IO?u0pS>m?!vt zc>0Z+5KLGM6Ahgb_|7L3rOH{`)^kE6o*r^wtRCX+wAGNP3sZ8r0yIX(FLk$$;^$x~ zvz25}yO>OT1TM z>M>re`(k~AiR?)l#BjpqZcuhNBAQz^H?eqw%+3Ioy%h>f+ZvCQym1DLqQbTmJ$we0 zt8wiLK$=&7x8K5Qu+3)6%k6iMudAzlxG(^EK!v|%*kziPp$rwA;NQDHG|I&MHjmuo zZ(c8hujHS1G}OGQq@>$IDXF+E_}zVfIyfJqvOkTku0oMEA#o&R{e+I%2g5x(Q0Gk* zm2B1|b{f!%*(4((8dA5U0K^Qw$<}zaenZ8Cj^~nkZ>VYiZGLb3l9$wb^FWSkdn9G> zt;PSxu_1YgN2t3)HDLCUjF-&a9Kj@|DVLj zeCX}VI3=%wYbjM8_0u8~U9IHp$-XH|cQvc%iDRqYflj;7+X(&bHA3mva&ikfYtwq` zYfpREZ1H>2;SVo`a|;fhHGeAd5^`MgUG*JHPL4Lnu}2VV4dM&zXU9YBOdI84=GU=) zDD@n4TKv4bz2{#wBkf#qZ64rQ)3fmy%}!_+t-?G4`BhhWI|G&o(UR#9<2 zm}EdMuMvQ`(b7yD3bgcWev|VdgOcPN684Gr*qqLsWSKA!C$l0kq2OACCLhrpi`r9 z)j)!OSFmerHpd)5vTPLwuM*0!ljLPE;p)Ljq|Z&SR86GN)lLsQ<|?#VKltW)e$Duf zN`;Ny_RIUsvVOSefqeMzp)9(Vs1YX>j7qup9nzNd^ zI&s?4*maI^j32cVux`aBu{f4)g)i2xYjC^zQaIaYGa(KpXAQg4anE` z%h&ZStJ$oC1CIO^{9msYudbpN7jN^Xza9@QN(HC(1;2hh53f_c#LdpkC@A)RPiEB$ z=yiW3@f9dng;LQWmt!FxG(FwN7!JKRxd=K1t>uQ)u(=YDngC4LlYQ^Hupd2z$A+fX)lMj!Y#M0H z{;tKOv7zwg6=xj_Nm>oOu`k)fq_v_gjR-cs*ee%{h1POX{-bNq(gYWbsZX@Smu$}I z0zXiRsU#_#xjh1jTu3;ZBaH{EwX%U-pR1?l?ovr|Z*DjNMOViQjIhl{8WZ0gQk*Sv%(s{A=#Pzt-_B+fz{W{jg<)Q_{pKm+3#AW|I* zydQ%33u%{o6#f^GVle4uhG@fES<3c`G}V{#p!P(FU0jJUhzTjmn`G+YcSE)Mo+{zu ze~6=nr#SU&+2#z=jCNJHbH1FdM>6%d>s!pr{_`ieWTf-;^{nC%-WolUQG--KWOcA< z06MU$94?+{xn(n~P-3iTsZeX4L(job=J)!jGNoVMrQwU=#l5d&{1}eOys^Mow@d&$ z6sqG1L?F@c^bvB<8n}x4nNdkAUI3mK?)T0gT^;7%;4+$fXRs0cO|}Uo7bTtZ>s6^{ z?ty}?s^I>{gipb8X1{#(**{V>ZHEmigEWYQk2)h6B&hl-*cwz$R9Hn+^_2cEcKxY0 z69;PaziVS=YPVC{wr$(KYTLGLw^wc3wymjcJ;{A%=gH3hXrKR*b+VF`?;BVjw`+|$ z>S_v%^p0ZJV6=Ddm`}CE5upFWuGksai?)R3dc54)b&EdRlKNOLtm_N=1F2{#nr=z| z$qKMp`vmAj87C_>k!K-dFBCBDhV*p1*Go7%Yhb4=%JbExc>lo6aZgWQ^X>YzyXx2J z{&aFr>p{)xRKnKXr0D)(D?`RK%?)*;G~I6tiRq&_!Hwo(N=VKKKkzpL0ZvPqA|qYl z_POqQKgyml<#@3nfz@>_fB;O{#5uXv6%M*XgHq$e^vAd=6s|95q}uiCsmTTNG~zg% zdom-IUVI8Vtx7Cy@t}!*A~B`=O|m`u`ZwysLIN^w{L%UL)A3K%y{QrV@jgQ0TLP4s zxm@Pz3!7WNgBbxn$(3EIx54#?+9PwD3(J^TDnPELK|te+T^O{}aHxxp1v-T^p8Zce zmIUYAqzEcgEot=6q+s!iUvC@ZQLVqRKFJcfDjP4M%LUn7+EyGlpv|v=T?{fB+bH zy6vBT)Jf+nfyaM(2VM_-cd#eU;%RdvH}N|bzePLI*H^0tW%;=~`CU3sp4jX36OD8M zkDVr564he1uf-Ti)%(3e1)mSBUuOB$;bWbQ!DTB`O{csTdtJ;fpBMC^YEs=S4-*y@ z^>0BIt;x)~bV9~`bz%NzH760bdEwxA0=u)cMTtqQjMa8mXJz{G&(th@#p-$_LH%>B zPv$8ob|d(xJ!3=@RG!s#U0{j~swCE&T|^rLhfgOix;@P1c{S>ZftejpI*jFM_$Nph zR?e~NZs-s1pSp?x-ZiSaVH&}lf#rAUPJ@&TK|R%)IxcCjthdk_%z}*@1zK2?N}Efe z5H_;SVLN(#QAs}HveG#7y4HOB&Qy!M_2sA3SZ}5(LR|E}I3_%&A`*ogMtJD)ZlQuW zYAImaSJ|CYjcBCp)>j$!ElQ)lX^%VWTnMe7YV~G+8b6==rkGIArtoi^2)2MYrIwN| zpT&ehPa)srG;T_0c!rJj$FU1CjqAVFG8G1u0ncgXC6PXZq94D5luI)NYxD=JitU*p zsG}s77Brce`Z?0NyB+!d&Vcz9z%U!b@@*?9PE6~O#}H0=N;aJGiX}Hb#e(%6Vg*pS zPX4jvcViC8Vwa)cQUZrnnun~SA^X!mZ;8@hOv??)OIBs6_r5jg!<4{4fs0d01sGr6Eo+5+!1PQYDV5H-;ba|C!I38f4O%AR9wgE_m68~81FKpsB%V#*-s;#Z`jx|J6smU9ec$Mj$V7cMB*nGdWG`r zq#HsXpFR@3cmjyKBylq=>7K?UWo}l<-){F_$%;7vw&<;6WDy;5*ZApnkO1Mun+(PN zk6F@jD5tl^6J?n;i-sD?NwS-w%1<ht36cv-Pwy zlvM%DQM^_15$Dii?N`3c>senSHL+x_A(yrdiuVS_!MSX;RTmy!&UNY9kCXSpA1(`S zwwaLro8VNO8WlKK!wo!bU`ZEI+57?r^RW7P#r7pLbD*;kfQ|~;3QShrf*=z-hvt*Q zfT%81BvTu}8}iL!unp6=1l?)*oJNJGouyOLswTKS-k62^x|dx5@M=3Mq_9Afk(H)< zcX`35#4-}W+Kc{&FGhRHXRl`G2uA5ebCxtG;Hao-c0vK-wb&uEXMIf-_X?33-o*Mx zjUkal_DH)-m&p@4Vr4;y{vb@9GV=oPs7R2?VspYMMVKLB;gaXGHSlhs&-5D7Z3r6qcl7!kxr>NhJWcFV z^A>Kbl4)-DD~>lpz{wYEd7kLAqqfGDG(iH z*}R*@43iYb+C;J+8WWwp(f1%GL$~J5jBRlUFj9l`#r43|f}n zz2HqH9BS+B`gUXW6EpP_a*kqgs;(osTx#o_YU}HM*BYke7Nb<$bFr$fp&RuRGw#T7 zs;;rBuFo6)dk@5`y2P!zTyq6BH~|}&zo+Ua^pjvyRddmVt*L1WO#M^ZXVgfsrS0iN#@ z3`86c(Q!mnj@J;5U%lyhsFk{AyDXCx8U?i+G67Bu5@{V#v7)xnx$p;8_QhTkWDJ7 z@PNiBZxr8%j`y`^)|cioJO!_TLhw&*^WXVLa9!xz#p0t0EAjJeg{yP|nyxH2jdzDF zSUr_B9B-CSnRwoM@PBWEY`#rY9rw`tU159ICiL1UL8(+6kwqkiMKG?`&urA`zu_%@ zMBb5iKK3Bpj&Mp*b?d9bOMGNJAdS6RG3#_I^*Dji03bPeCHaMI9lxGlr*+#30y^fs zv#45n)HcOd65mKvm%|Wu;YsW-(_pGhTXaNo>4qbaS+KrE&QkdT3txpu);Yf_wKfNl z^iaHfTP;O}ZOMdRJtJu*kOEZH20tH-{9ojnfaxnPsd<8EK~vHgFS|SoJ1wRpvzwkH1#<)_$N{L)eh&mVRSn5LQAA zVXfJUrWMo6!i{4FR0Z(MncwmZn=K{m?DL^ZpLS5tqHk-#BO>FNi^}DFC8{k4qO%hm zu{)ORM6AjZXZh|@TeZD+pt0Ahyy>=5wKwQ#L#yIvDnxU(q)8+rzd%p(-OJ&j1!sI0 z!lC-QWy5YF0!p7A%zwVcdB{ZAj^buP;wonnh(P68F-}6&Qsx^H#nfmG_rutpyI-1T zHV4-K9)}_S*c^>G)Rd!DrJiCE?Xxzcv<1u2xCO;0e;ql$^yu4~F59kAyG}Yq%|7ke ztEEO#j|DcBn>YwdeH6u>c$`I+;k8wot_|Wp8Vg@!Mxh-X<9LQ`$DF$f3|_j0>+)%+ zN|WGrPC$<=cOR8@hyI8eJQw4t6i4md8V_-gO#RND15dV_0&RX2qxfsOL{Xtx)W!t- z}f97D?+gR2$NP~b-QqD1+k}aq{Nv=>4 z9>vEc-UgG7LEi=FaBve~@ZWKbnO#_dEg7W~T+; zj~(CS{%zuHwFy@0T_^esrm2-@Zg6`7?Vg5ygXZ1fVzs_F9r8f?R4Ke8#Rv2 zYUln0U6n;x>*4Q%86e{K5R$taM?q+j0@-Iy+eG?#8s$ouvnJQ5DHTf-hb3dfkYp_N zch?eiH>T;0tj3aD2EIQx@0yKrTPn= z+U?W`r^ZlX0bcT~E>kx`@fiY`f@0WJ2A@TWKc38y;~Z6Vx(LRjK`^YjRJW`BeZkLU z93zZOz?rltXc!62sYo-?s{Lh}56bx+fDwRhdLkv{(Duc%|nZ;|3Hj-zraa@QZ;w9PvHf z2rMW;o}Nl>Ii9Y*DzYN!)?(eMt=ppH^yt9pa0w%L%Rmpic+B@q)Kd_mHsy+R>SNU8mY z*%c$W1U4UHD)fxi8qEXRgUv+PHn>aqXa-k8T=SVezJPkfqOipxGX<7Rmyn@C-9|O$ z%^j+=)wXIPwe(5+AsFh5hm)LK$DVgkZW>!nQmesijMUWZ-Q=ZRZ=~;nP>e&-Xnk2B zRI#G0H|QrP<9kL-`YiLe0PSF+3|5gLZ_Kn?q%5L3{0v&-s?VXC!R(>*8d8F zD)(*c;~LHRI-)^?-jl34OLWWQ>W_d9aK?OrQATf8noUsS3d{MZenw`52!l==#PFcp zTX1WpCJ;jCdNlSy7PuVUoBd;3G;6b})U>IZ`JoLW7928@00}An#g}#~=CQB>}6L33+H*{@*)2R`>J#d!jziE^}}bo&*e5Gaa`YHw14?C=R)!)rCH*R`2U zPH{0w!o@SORPT0H#QHk5uKN z7mJ5|H2SenR%&}TbnC}w3>$FDNmn?{&FDs9@1wGUkFPRLk1zw9R#UG*MqK5GP*v@} z+PsNnu%Vg~G=JPrvomofKK@}XgC%007}k@P1OvDn$9@@O(i`mt0y^0i#W&*-w-ZGN z{~;D&CkAW=QhL&A+jrasBKlk2)5XIhn*aqjA5+q7LYx)}7Wvb(3oXNB7k8#rCk3VD z&bjiZspQYN+AY+spypVg``j`puJq2bGUfiBT&CISh1A{r;<*6%bh_W>jj5Gvyr|iJ za+EA*e~L}KT3Bb589!es3BT*UpxRljZc&%vVFo9WrdUw{_Jz`GGjHtv;ET=GHP8ak zZCcfRZx909A)^;UB6%wFn{m7tY(#8AN(uCT@tHf_AHDR;%O7-C<5+f>I5aVIL|U)R zMe@K_OpRMV*w{a`h2W543s7ThH{{^d$FZ53q8)XDm}l!yJSR{i8W0mb#rHEOeHgR4 zb$ytTE<2-VFPEhso0~1pPM7oI33k8iYBH}zzn8Toa1kkPrhDJNqDP9D^?w)mZKv>o zM=D&FPp+P|Ba`LfVC+M!gL`~E4}Sdgc)Gnk3n$`4C(7AJ;My|uB#E_)Wa32s!8`oJ zxn zfC@a~fg-$c_28L%!VEm6 zm4L3d%lqq_@SlG)s?1lGDcr6cLfPTD%V6|nO=b1$0DBiBaX-H6nonISg&?@SB@%Q* zseup*{Vy-8KO)FpF>Cd^4@{Ervaq?}>QaG5Ew5NQG_Z*%AqrmIB;g&CWpN9VCZXw$ zw>2R=61&fGga;3JoG zQRfmVQx(Y2mA6$#>$#%O6?kUydr|3gyAFcNwk}Leip?iTd(rt>G4Xht`O9DdH(Qj6 zp6v}p?9eyMN@#`zhneMNbBNUuDZ>kx z6YD;>5HU}F#FK3`cf-EMyZAOIIU4?p`SRf)nLhi8$y~p4gIeMHqU%Tg7AYqw?~|aw zyYl_}=xEl@<7@Zl<>+WM&+*y)h0orjiTowpz!#cG+H6f2N!Go8LJ>ohNV{NP6~~-i zU%G_8_LJXQQ%}!y9x+#ayjWTp%X5_I+JU6^BKz55NUssm4D@XQ;7C)z1JHoJ4fmyWm}jn-$uQc zZ%=x!-Nq5M(-Ex^?#<1DaUcm>SWD1C_ROxh50v<< zQhM+?L&a1gN2bMTuP?~SDD&;_YtXn`Q0z1u$Ch1-9SJh|z+lHGj7%O2y`||+-s|Vk zLc)K{$i8hKMT#*#U^COee+1`JPRaRk3tbtdsuX%W6#;RI zcLyQ2_rLB^_5`4$Zj9EH%SM%wD5M9ituCs{df`W==xp?lS=_U6yWKQflU!6^6ntx$ zx}0U)hRT8aar&yogfydgmLrHsEK3KR%A+Ch*$@th@6WM%{MDg?qnpy4Q?+ z?Nm9Nh9uLi`b58CeRbETwx`2T8owVqLBj%$K4VTy+_tWqsVYNYic9tciGgNKk2+^^ z7Q@mVf(1fqe=%e5DW1;$2tn`z;}F4BSSF>!k2UNYMCqS|GkFW6FjGxj7r@S?J5vh>- z2#j7FYyJ)LHCPxrHsm!%qHzyL+t5}k{%CMsVz0`{VvUrd}1b1!%zs+qyIO11Sa_pXI2Cjy0nyYaSeu`G6TJn z#2e9`uW-dh%+}4Bwy|@yS6u@svK0w(c(h`inG$-Bk_R?#Y{a@B0c`R5%)C)13Y$Ew zwpKIY)A9sWn*&a+Htv3I#IbhB__ynR=iSH ztYX%7HLqM?^XYChD$8^sCMvIwhN7|~3_w-y%GELM{mSDYK!rO&jgxh2yy+`@QYFz; zXB_n;pK3&G5x!$3Pvv2xW^xHb%4H-E0nepu1@>_W&25y$DaYe5exXV#%?uacU0wT+ zl=LMUU@BSPFrPDvyp^HES~*?)+qSWYdnt613R2GX^gCtU7II^xV_{Dxq4kE(nc?3N zt|GCM@8PAezV=1tyojAfwa@KAAUeBS6J@`ZD69T$WSI~WY-zCtra@DP$R+MFpTywi z(|obfM|^?xI1Y6lEoE&VkCipiUyb7O?hFPNv7du?J+Ze38P-DMPOe@EuD8`BGA5}a zf|3aRqugmvpzT_7>inep=$AizGzY<@ELZ=>oNj9|U>~RiN~AcGNg`Y;05&d`-vzEF z2YjVPk*O%#Gq$vhX$8dyK4*26DphHr!meD=P>J<9 z(jwZY*1S%w4Hmj@(eN%TtDgJ+jj(oy zNLkn@GC(~wXMbn>eYbPk70%oI@&J^8@gDB-$(#I#_ZJ30rOYWQPyF_$7T6QctQdd0|l=CCP+r+;|&$VdN|rt zS#gMR#Ix!4#~`S-)mW$pA(}l}8&J`&0^2<7zSPU8%CWi*Jh$miLbiBVvk-oE3lpQt za4K1>qn8XkR)7pDZ5q?;$C~e8Z-NFY_UdC)Fg(y7p@_Gw%TS9|(_UCvQ@1A-D!Cat z4C21B6E&I$OoQZa9N%MGjoC$Ma?gThDJ{uzaIu-(ng!O*9Vt z`o^V4x0rk*A0HTHZ|FA}Sq}ny!=tz4_*-cl^SDg?VeS+^`$eWuGxu00@9^ZWa86&r zoO}a=@6e59T;fBM4F|G~d$J8Xvd`d&5G2W$=z>vNS7b1`M~3K~(3%(bpT5dIzW-*W zk-REaOzd=cCa&D!_RSt@`?W zW9cLbK`P1%NLt&_6{+#B;CXmDDIxV__~noheZ4oH{C_Zkn>K4Ob*m+I^s@m>A|jF{ z?udrf)qBGd`7W0;l}G2&kK?2#QZ>K_q->pTd=LBVO*w@`>o|OrD}noOERP0)2Nvcp zE!*cbAF*$BM1TLko3)8mcA-w0*D=T_Y2P%1)a!-Y{1nXgt4N=3v1}hfxdkOQ6DEK* z_gM>0q(?jm`AA;mXC>~LkB+LA$Db-Yi-0>y^bV21eDJjWuf4Tj?hjk4f-(ON(r<@*qKi*@F zF+G*153@hpGh6nuKi0CW+AYhhi+1l(tOAW;Eo&dV*Vb_!Yaul2tW|rt8@t5qkeNkp zLUu*nfrJv=^mZlSXFB750^}<-8XKznt~f__pl%{EGjSN}r)_pHF9IRy;hPmd*uEhG zlPjw=iA~^yy>Pkx@z_2+AUXFEJYIu8PZdDg#K^RmD5Na;*<$Qg8@!Dl&PYOEjw-#H z4vpZFj{Z(z+ImC^t zqHIReeBG7U9S0vXo2^1QRZSiSlNdKjl{Hm9P`JgeQ&}UW6F7;xHS22snhH11EhD*5 z54jH0L-#IT&EH+$^kT5DKz{gYmOy>5;^44uW}FoQX=gZ%Nrala>I zYcCD8)PHxhE;^k77{q%`!@W~|o(6X`tV-Exynts*hC36`<`~W^X}PxlicTF3mMw%d z(-gF@dDR-MXn8@lY2H$phSyRpdUMrbn(5fb*|@d3sAt9mg(Xf1yiK&Rs=!j=!6-?? zGHSLVih&$O-TV*>8aK}rKA0snk9o%fN&=*#aof{1C~$bQZRdAH>BsQ2hlZ)9UX4mo zXUdRC!825oO-xNN!;x<}4gQjG(~ir#0qz#le{pb(bnO38`lY5)QGXY#+cKLp&?Qi= zJuKo>iO+XqAHR+Urw?PGurczFe^a7U1(>lzFjlRe0`9!4eL`*Zlj0&gacOF|T~r zt4zqAA)}yE5H9Os`B;KgdDqJm)i?ccMUE5WA!!DD4`_`;&jN-y3@&E873c|Jui{)h8xzfr!ikzP)|*N^REF)lCXn<4Dy zb#NWSJa3DikDHsD_v@S9uIJtU>0R9S`OItH*7jCEPp;Rz)^Cc$(e-)wMx4vJc5l9C z>H~2H2Tx!mI|uO#W<@u$4Zl4Y&|ZyXn^#}bx3rtY-1H0~Y5kM_7U%Nb>*-04*r%H^ z+Wjnj-TWeFu)c1yl2+HcDb{XE0gBZF`>iCOL;aUs$C}y`8sIq{$( za&uo18u6v{`iC}#WPsgem`x_1$mqo6@7&3$Zf$i)k`E0gwGGBdO^W{Fe&pvGgN<`D z(OjB(X|CTlV0US5lt|D9A0v6&t-pp+L=V&mF?vmUeW-A=?JxOB6glQd6LBC5U|Fq+ zv1;2u0{v|~H!bG9n9qi^-?L+1bvOrn>gy&nN)244+GYvFnT%t9majd?qvaau zY@O`Y4((ev$H$??PtN!~9l+XYwU+27V}T`>6j&yS823WnWk_8^xXf|1t);A}Hs}GH z7XENbN8`4xcKRy={x)TG<=^Yohuz$yfyq>y_G9@rs}3@=*ek`OoTda@EzsBK-3Sd6?OMeE?&2Y)18R4%)DLO+6x*N7+29+E&HbS=2F%iQj531~-t za^h-}#-^U2{A2?&D}>%mpL4LOg4lkw0(01BuoKHidyyCXjU612&(ehSza!a zE)OCiqWqN#|J5Rv5jjTz`Tiz3W3_=w+$4iKBJ6FYzUAL;PS54yl?E!bfN&dsb`Gmx ztL~40rX#$nq9fH+xaF zlP^8UIroryLLd@CzZf~`s^einn3+d%^Uitx(r3ADkA`q876{HR)_}EM(S}qrIoyol zgHw8-k<9I@gl@b5Do&aXsS6Hhrz(#@MNoHjYZIyCgc#AMcWMva-N!qQKoE zj`uq07sOq~j{-)C!`}6dL|}bNy9~jtN{^SH5%_Rsr=}Bo*0dR0Et828X43Yft%D!2 zXu*wzdFV9WypKjq*r;NC^WkL8_*PW`n{B$|u}D}o&si}l<1oVX@`;wMA%3G+d3t=?DBdxKACX7-jEHeHYJqs%P3!W7tv3*j zMu1n^q|eFeOdl%^>BXA#^1!ejti;DFC@e}jdQjo!WwsKA)y=PwcS!nPC(Uw|ULAUf zqjNFJ>HpiGy4TqP4POlFrvzzpUyrAddOB4OOE8W?ePMjoI~@Fjd4tABVl9y!HTdNTQT@rX zq7z(t43c)jS3#E`dQ(G5&FHDIot&#ad}LQ37Qwe!w@v~nv^F-6H8CY2yBY>!aOFSf zsQ}nnO%t==()}r28Z`-ZX>HbdQ3_iPs607=bCwnxyxcLbbp?-pPE zcFZpq!lD{>`y|@CEho`j@57VV2%nRyPUO1+9JswJRBdgH)n6a&8fz$*?Yl7lNZa-c zn+gmwm=9UtsrxvJ!ni$~4C%*^17FXG8f|Aq$6Ps@vUy+dhjUb7W$cfTCnRSy_Smjx zp#P%wqSY11=$nfXY?kernRQ|6IGP?>5Q~s+MRh`{xiYLD!PH~yNJF}MpiR{1rb<;I zWnkf}-@h*$a{^mLUE>3WSi)U)5^L%7hXIo_a_$pB_lxHL97NOyLT%73cvLKh4h3kM zp@xd!=VA579H3VG8S4=fENxFwG((61#w%$s;F36CFEK5B@M4y7Y|ScSe%-Z9 z%^MW&4Tit6MXD|`MXfYms5N@qd;aypc#r>G#rl-9u>4A3T+g6gQQC=L3~(F;bymJh z@w9sRH}%Ms>Jttx(a@2Hy6OVFQt^1B1b(Q!Y4#mDSp1S)}o`MOU&<^YT`j1Ad@wA*C45;j=xyq$JLS@$vh zt0sDkUh6uupmfy}e#cZX)w=E(`e3MSS)jVvB6^@Y3YhD5?oOl2hRa{Obn+&mvD_EM zh;>uItw+vNtl{U|zl0E(t@~N!kFFGAU)ECR%^Q(KGeRZnmM!FF>-hN?a;<{5U02pq z)TIGyj$GF)8at!xa>_rFrU7@0**eD80`WWHrBzOGY(3we|Da&O5N{XZgkbOyQ9*yGW7)KTe97SA}#KyOfLdq*%8c{y()k3j?|iv!dY@8RTG z(YY(z%IJf`?v2;m)|*}|e~{r7#mUf`MD1UB_rsf9Lv=brsT$Iyenu<1m}4ZY#$P@AJ_c9WJ4ylJ1!x=6+0!t$!SDsTn!q6&nM zZeN?c;pndeH`oh!PUX%Advws(Q%34hh&=?O(>_?oX<&(vRYFCWD^?~-BZZS(If_P!$Lw`Wh z_+X^9RiK1ZlJ$kTUDrtBxI$BJLL&dI_DO&c*f5etr^w9Os%2%Cv-B-?x9t@DD{9!8$YrnD35To!WTwU_ zWnoyq(3Iy;CY znqmmYqyG`FQI)xNn`am)s&`BP)~!Iaayvbusk`+JJ$h!baiW+fx0&W8HhWLi{e;Wk zDCHmJ=syyM(I14-BXx`<&)M>&;DA|udd=26uh;$6T?0fpy0K{jBBvv#wFrIPwd~s; z&;;|L@=m^#Ozg%wP_g7FG_BKwy$Ak&Jnn4FLuxXO!Kj@1m^aq<6yBRw#4q6Nr*7wf z+|;vg$IUS}A;l3_+YfeloB}(@fFTU4GW7&jb+N(Gd1snKHoQs-q(^sDSRElWto2nx zJ5CRGQ{%yuzYRLVmbc+QRlx<)GuTf)n`z>BaC zecZXcU#R1(^WgW*582A(rQhnUF9Rs`~k|?PZVs)Ar=+PHW}w4l=@}5eg(`j21JpcFh(dfS?)t&^E|Bi4;9t@!D%blS{TWoTwIEz z))@WH?uk1A#=G5H^gGz>t;Tfi1Q18c4EGnD*ecbt;Ekq;%$dsUfG?W$C1KhT z$MXSAS$pY&M6FdGgl4GRIcb(L&`X0m{W`G<>|g!}^=wXy%`R|D3y~G zmk2!NFKQ!sD$&DfX|~N`V;OZ}w4MbQ*S^4MoSUUU>_(36>OR`7ynhg)A`=YTA>$if zV|8rQmKww3AMJ_K#)mP`VHOy2k~&y(DhGozn@6{X6cTS@sb}JDCHs4-t#>)Pasg@) z$Rrhi8ty83J|MML9-qIN%+!%*BoliFk*4>Y8c*xrkJkR6jdi4IB9eQ%^N6aiB0vi% z+MjPY+qu%+DHINar@r@CM&m#|1b!Bf7iuu7&S_Q0?1A$5XH$KIwd)4IO%oV~yvLPE z?WI45@kab}z6E3(TAr#blmD4G6ce)TlRI7@f<7ae7SZHUo*AeW*5=@-U|2*d*X}du zz9(GMg%Ty#H7stxd>nb{O&3J{FqupXIMW6={LzQRA)UKqvT&>t&Aeb=v+b$XZ4SvC z{eux!UG$Jk``dN>V-f!eA@lpgkba-)Dk0a4MrUWuD`2^$&pq8Tq$)d*Pj}Ia_nZkg zxDk_wSu45S`V9jVbQ3QwSpTMluj%FdPeJ1fNrZ%wo0Ivm&Ly@P{Ow;hIlI@8DM!h# z6@E^?eLC``K$m|jp&2&@H_C}2*80y0uhu)sjhSH2HrmWC_|H5^#j`G{PoKq`VIWFB z_|ym}CkQhuw$Fck$>g*lP9xSV7brV=|qSPf%+!?5k z^-sVi9IZ`YzRDybeFJ3f3(bMCb}HkW7Y zEfvu)jUHvUbt(jjtWx^aFKW9&C}f@Di!%$)*Sfb^x9F&BxUtoxHiw`f8fw`z&3Beg+NARg&29MuIlt2ffFFA+wUsjf(=?Sp1Pvpp zf0hG>zOI>}%V^C|xu(=`2xEJi+5@pp@&?eG0kR~6Y2oq{N~96Xk6$AkqiqqO5V^9_ zoY_mZt+W;nPU2Zgf+KR3Z-H4vxkWT+aC`?xBIZP0I)_H;_wb?8azX9uNia^%u;dX? z&6XYeFV2IFX6rDkFuo6xme6SBVeuzXsfjNqt5FC)Ck=^~Z{7|B9ceeJv;%)dDP9Wh?$PzS!bo@7T2q-0inE z-00t5h&(=TcHee)+BgmuZF#xp(=E`efm*i8GR>7ykzBPLYQCOwz-yvq;5%ZEP_Y~@ z;8q;6=+slHbIsOVB!?*w)>V`(drXjQ)S*y$gibFC9{F~!p%t$37wya7UfZ79M#yil1RhoZ`DK1iT_f@TJ$UkUIk4%HA# zcN=czhkMoV+HOD%x(Fa*4ZV4uxwU7X8pKo}* zd%XirU~%?I{LFtN^oY5?WUgYqBWFCP**-B=ACn~3SeNVRMLON9Sv~OA5;=lAuTYOK zX;R~s8eu?m|YL9oc53$G%_wLD0Vp|zumu({@v%|EW=CSPST*Fb-7 z&O1jRk_`5&n)qb^S*e9uwH!dM*O?7RH|YS&01c8n zyRkQXAC7S88_nKM*ex0Rua>`q!u^1xZc7&M6cP`!I)NS-v2I`7)>k8#b z&p*uS%z)1yzZTn399D9S8a7{neyoPTIMY9@-HcpAyV}0e)S(WhD!)RlKp>)S#;n^< zKWu5yCJ^w|o{mCo^0II463EjIBP?zFc1K#zU0m+3JGQ# z$8v;3h#P|S;{3%DxpXo;M+dC+-xxhSML(Zf0| zGN?}KiFrlZ+6&NacUiP?3zQ1O#&N$Uan9bYd>wFo?_JKf@Jysl`X!mBh;nyUbvBMROk%KZ0KStZ`45K|? zP$jJ>EM(KwW4k5qR!rc}W<5=tArn_?lrhf8EaG)Ltlp-V;2hbmkDUqUQ^SScKx7;XN4{lcp9MxxxP{wsB`g; zpa4ifx4#gnb*q_*Ln%vpk#<^k?E9ZCgCiHtK5>a>!yjFDXd_6xVLWh_>S7H_z1ZJR zmh9eSi8lhTRD;_by9wMTDJb%A zVeOBL^IKpO{qGM^NB`TG+FE`QjAc0Rz9pK8I_3~Jc(hV^LLAzLf}4c2F5iL0sRCjZc(u_qVEI0g7qLY| zCwHdlL=`LbJkPW91y);=#0)gB-Dd{9AaXt9P( z9Ldm6gg9n*jxsZ+FQgb65=K&MtLDzpTBjEXd}cX|@B&@ECfp>%gPYeUXF<|()1ZFc z)m}mCy}Ny0@}~z!tIV^U+`T&uR`aCYT*eNHa_S$)u@YW^ad{!qh}feVsw0lhzZjy{ z!l~K1ZTk9!FwJY}dR-(`cZpZqLZ}k4P*`PeB8kdJzh0V(olL&gKr}=#)^kmFADy&E z_fXLpB^;6RzwbDU88K9e&Aaq2N_o&~jpho948D6iN=oj#%N+LDT) zAzfIqNpcd|<&W{rRp((#J(?LG? zucE3&JxnMesXE)oc3zMd=r5B55DofRX4;eNHWg95gQtU@Z(D9bW&uW+-z(4{==o=1 z&{1GNa9sta5-||paBQ9^b5tPjX!&iR94+L1we<%2AjfF+2+vTDtYBA+T#D}uk_#X5 z@q8kVkLg4IepAj8U+y_n%n&bTj~2hiT3G8G+>aF>2Q{(M-BbUKVEP-t#N}(cxt8)< zK{L#MoC`)|MZ+CvF#sJf!UaOJLPS_UGHBS`R|B?GR8YqjP$R@74gyt$d_={9$_E1! z^sm~JyB*L+^w%DC=7aEAhUJ8z#4=o~LR6@V=lJd4g&iL+be{@+Z3Qw#HKY!eUvuCO z1RG*EtT_+>3T+Mdb7F%Vpk*=@yTja83WKQQfizi>7^D!tKtIhfXe`Smu_355!wRDf z>j)F78qYBtfL7A@_EF$j`t7M`i$n6BYsPu56y>6Au1K_%4GQqa_0wVJ{LC%~NehvJrxQ0jwl!6B^;=zPllE5g_^oBzj zSnZ`4Jx+1lmasJ`B`-b0gVPU73)cHfZ<67O1x-dnr(z*eJxGF2Tq2M;!6=l-qbpdV zUhC4tFC1e3xK!Wy1dzsnNiqGBU<{N+l}H5lLltC3h-X3YK^5#Fz_rADrU*PDpTstj zCKd}IvH=Z7WrL1`7L4S79StIgf#ho#{8cY8@c#n?K>WX;N3mJ#(XbX{V2m7iq6o?) z2w}7VO%z3SH~6Bl&_yF)i{c@R#tkC@Gu>FYqR|)^V_}M7A&OeCL(v@`geTgB9Ezs7 zA84Y{13M0qC>oAvBxV;0K@m9^9P!I?7>20(u4tMA5Jdg3LeW(F;D>r4g`z3$1U)no zc4!O8p>gm#W1xm&VTMLQ3~d3q64SR)utM8vKBH!5ER>K235sY&B#h7|Ku}B?L5AK758X!9c*k=&7C-9DbP@e!~PjFMV0evFidIA(X zU_NnBJrQmfZ3Bvi=?ReRW(h?@^aKYK0er^5^8|a(2jZ3np?N~vAAozt!SV#Bz2e!Y z3(0dhIG(M8ZbqF%VF;cr;CI4m)+p#51FREbW4plaj2g6&=XD&^P6Th+8D?j1S)QzdL z;6r#Pzh}AHQd9C3)A78aV^l!=m#k=>(lRAq**#~R4C}n2ABhJ`D#5$;I-D{_x>nA_oASsNidp;duu3}AOeUD5UO+Af}kx9a!y+OQ8ynb8CM)j zi3M;81AH#wq*W@i&Ky-L?v%XGd-rw}P>BvK7jwPNEP5rZf){oRzJrB)nO~J+Q8F3i zYXIK{(l7yo2k>(cQWKv`_XVac`eo1+z4r^#dgb~4Qqtc$| z@Vt7}X2TUP4Q(${v2w+8nqj7aYT{dwGHjgIM4orXD6Efm8UhXFIhRXZMRpwRDISbk zo7W|BO0~DKXI%}o78q~H>d>+?8Ovl?jYmaAZvPTW4qS~Hr z(=v?-)tSZs&6|?u5};n-cBRecI&rDnT1-FFY%8>v_?)m01x1OQfi>*6*Io zIq=AJ*XsMDzI<0mCt=kY;la?@mVHn2F?3RJk%M}iDr#Gh1xy~*dP=Wp%;1B zo8oW~GHI&~Fb3eY0ne-~l(;d=g3;E(ISMRiX-Rcwo$g$H`yyojg3cQ7G_WCS0skHZ zRwoivN|A6Hj)sG|GLsJ`Z|&I6iY0Zjh^+|=>`r0ppEpw7*t4d7X>wP`)uN<%ol&27 zxv|R+pqEAkp^ghc9Tj}~$p)V86m%L;Iwh$}n%v2jSv{!B87kV&{p|~FKUABTL(nB| zIdl!O(&kBN;6hCO^h8R@x&+C+ulegc(j8S_`gBVG;FE z)HaLj$68eMOotPeVWx#HN}Q+7?WztLbq;}fK6b-$sR!|L-bd4wtwz(PnUd=D8{j;L z4wHmtNsZBi_};N&>trz@ge8}lFU8WcfnF&2BS2MC(2yq@mmwETtC~4&Nr}P3%c7|0 zm1SI+rWY8tLo+4OK_gKlt$HQUkVnMIr*RS}N?)NBHd@qFdn7<~q^PN0 z8LNXwi<(B549`}ysM!dU0iq*C&3&Xj?nl%#qx&c%{tAUehwrxXb2VR!Js0%Gs`HeU z*&1SRP-5E40uwo|oiU!v<{{%#PU;fEm0p-%RI3}mSDba&#(BAYZHPG5O>ode8C_F! zZ9zPkZCfY6)CPSjU2XI-JO4@UVuR5Ni* zx<^K3bybQPyWuhT6`iiOp0=hlBOxXz+7&xnqw(<2lU_B$8lidN5wI${40OB+>5D@Cknmm>p!x>wRn^#&J zmE@?!kdJg#4{eHePr@j|!zYM2BlbS!p%A4l$->QJ*OCADxmg#H+efX#`^ILZjvhbh zX(a6tLlB#xGl`4&K!G(XDkVDYArzmaC8ge=%#opeAyOUTEYMFk0ir(~*+i6qi6WSP zOwjXG=U@#%69L#qy3~=lj{go#hVTUr_v?5O28T@zuzv)MZWpl*g6nxRCV9lvzu63E z?kE{?K~D(JE9OoVhD*9J;W)Ut5hk+DIq#>c*|1UtS499paw%m02Hq4s=tL zU_!Y&k!EN1&r<}Kb!PI?riFMSh$2W%Y|UKrua>cl$8a!9B7(Rr2#Fg7$|KYes$AzQ zR<*L&&cL=GL6pi4GGhNWw@C~UOb!R4aqGJudxm37iCEkmru$QP&74s*(K#ROYhn&t z(-30|UQ+cU2%-81=1fr4a6IW}F{y^_T&pR$fj|L~Y20Rrr#Z92!?wkNbH-AsQxTln z4sS)po|CuiLGKe589PVNY}O=j&1=~BYJnQ_4dhWK*cO`k19dMsTEn?pKi3vH3U zL1HMl-&lGe$5joxk^is>|7UDIXG!&E)KZ$% z|Dn}wlkWe+eEM_W@(GE3_|~EQ+%6mox3=U1rn`q!)PO*nr{t9v*&nSUSX7c`S&N9& zLc24uk!bF;WaO>5R#Q`F?2KGNu2}O8oNI5zs}C%xt3hcw$W}bKLTwP|21f|{*TNBM zmZ(7p8s=d_kLSR+-LZ9(8>`WauwKO=NxOIq01QI6{C_$vPON4;$Hr*`mT2BONYsEm z^{{! z{|i=ffhe;+vwdfyoM1s9vu4HI@EyCU9V9^yQ8%hspO>c&C$~FxO1@Z|WOr}au^U07 z+o-pVo}z{c%tZ+qoF}OuzPjY>o+s7ml>8r73RreIThNL(aa+b6SRvMdI(wgx6L@U6 zovRf~QJOzBdBm7AJhVWJi-qUapZ?%oL_TjjG&ySJd}}IVG$V!rPiHCUqtweYdr-4* zCpw5!PGRd1%d`-c|4{3Xl9DZ8kKyjwa)(8TpMAF3N)#d!i}iI;GMXywuPT89Y%MS$ zJtBc1gzjmITMJMFO?0Y(EyV*uX~R3c(oP{#N$5&w$q1wIO{=$&dmyazIslE?>dlg`@B^4}+vR z219d%4JsZt^H7MHBd{_x)?)aKp!x zn0+vaeYq{BWi*=Qc$a^~;71oV37}R+;#EdkIAdBLiB35_#x-)B4}nE_7!=C!_>&eZCMfbTJwyAoe6PQ7iLZ&Xb|knc_LO?=EnAdYUe{9ykqMLDt>r> zGp!hw-%<;jj(bIKte_|6*84hYChQOf=1@&<58-8gCDOJ=JUs7%D0bw;e3)6g)ahW# zM&M|VIB3ELVT)6#0gho+AFl7%LlloXeuFo63Z$fLQ8E}C++MKVd})9)M(Uj1Gd1lg zm)fD5YEQ$hTDjlkl2%;Kxfa#hlOrL<8(_C6-rmHl#(jeBj&^`DIeq16E29)*LQFCd-+EedKI z{=`aQvf&z{of|!LCCL~#+G$SS@cbp--vTf>gSV5D_yPr<xMqMB;bPho98(Qb)h2}{>O8;a|3Z2zA<+uHWOvO73ZEVcJL zo@Uy^r3^zE>|bga$9KyX@9pD8$OOD5XyVx2Q7&$As-~Z39}93sl^Y}CcgEqu zT~dg-)h<1eJN#uzE;Nuac;dWhTY-5VoHN4{hI#%2f@_&*f)!>wD>9l9U5QkO=_q~H zEQ=nVolhFqM` z&hNVX(VDYbauZ0=zBF&|Vcv6SKRa_q!PioK{~x2y_~tD4A9&C8p&WTga*jGQcp2VY8HF9!j`#Ji}8fph#ZTz(CGO&K}@Y^a2bH3|bTEY!A~jB{$Go zGECnSfUdSj z*l>2hGp{4}8a*^W_^KG+vMi+y7C1~MLBYOiu4r4cg z`;>1Bz$=FN00wP&mwWHB_AMS~x#P-F++Ea_e%l!!_AK;)sJ>z;-sm+%qM>EOCWnPz znz)>^);b-wMzlAG8Hk9bKTPs{zU?v}-ZlsWv&}jmHbr67)kj_DW-Wac8M%7X+;5CT zcMQ&7m@PEKPN5IdU;PsR!b}BOErW4s0~C15@`@)kbEVVP3zIDPVQJ`yh!C*y{M=wGNT#I6SYygQ8u2cKJap+2(3c+H$si2 z3|{HPYVurG>McISjyfH^$fP(^?8Xq@G{#|BCipz8)qt|14@JhBJS*$@((~Rq_T+ zTcpIvqceQ#Pa5gWd8a*NzyPr4gQ*p-hO|`9kazU&a)%$eV>!J8QmSsu> zllk$nwO;8o#ZCmF5{FuP;UMJ`BmR<#Wtq-;G-^4hpOt9zZw{2(mbu~i=ppD5(+*6N zTMFoMaGv=f%@_P00KH=jd}s_^vaBE_OKVT*3uq*q_7|=-0OIm3pyE+$B9E}F#_%R@ z9ol>j{0$-0=7im|a!m>$F)tHXJe86!@|DlGKR0n{pg}_KjIwp4wXGwt;)oV|-lsjl zG2;8}Rw2coe0Kiy{ON{*KSi`V^jl9sRS@ z#;#93JAZcm^hEXf*{SNiW0`+8d3s7b*o-ZTx@SzbZ1b17hQNCeWvXMX^&}3GO`sB> z@$9b5c?;Xm*|84-*bTT8ctw7<5+(cH@S?YL25sj9jaC<)fh?%cBBsy7GEOrV zo<4t}oqm$9ct*=??UV1Wu_quUvHYHw0%Ue_PfL_HZM95E{`KVBi+4Y~y?FELRB;rK z5UzXM~lM$J;{RB59cu>rE{`fUNK zi%<&`T@gyxcOW@(V+YtquAvX|RYqmSlP@xn+^N1t8fN;QZ1prJ7vEizmt5Y3D`#TG z%r;5|HD$(>5(MIb!PT_UCG0~Xq~GLf#%9~PthN~IgJjs=QVzsx@EF1bd3w` zFo0RpaA^6$WNnjIXfg>Tl^n4T)gO+|$kB(nRR84FTpmry<*LYd!Yd_{an9w8iv+13QZlLUi>{ z=PA2CmnnUE22H?~;!oph_mfI;^z>*-u8kyg=Jwm(=cPCorELh9`JWt-6H!8o5}K2Y zQ4RQ}8O<7#2u{6VXE8w-Bq|VtL}(LaV{w=#3^YPnYoGI6lnFC1$8d{xbR?@P1VC(= zF+M7TCsCRSu~O9K%lL6JPawJ{|FLG|@}*($4#0BC<(+q+Y>j1(C-KR}ivA>W^6HCg z-K^7JXwbZzCzM_}v196$usykI^ryOHN-l{KbnM+-;YNK=Ueb!9i~}6VXN9&0q7q+b z0KWi_I6}$}@~k6wK9?s8NozIk_P`k+3F z`9|}hiGD`8pXQGgR2w<<%Z`lN9OZ8?@YdUUx*_96N|#=$uRcsahFrPpnBVc-n_k}D z#p-UQ3L&Pf6O?kjq>7P}Rh?D52-G!&C9f73MBv*>a5Mo_7eZb#bi1d|O~!R_1;TRe zCH4mV45{X>1ly?Ml(hZgM^e`;eAc{`1(K%4i-h2}*- zTg|aA*ea4m0H+z(Kl6m?9x_Duns_!=WRt=hxEbK#TTC0b-NXqimh-qwI(6%p*KLdG z3sx-2m)BOpcKn4tI|4LYqJ8yPl8n!p2e&Z_Jxt`h5@kGU*s*y+&F(uh7vzoeEhUM_ zGL{6b0%DE>VM+F=ZaNl8PVfE=lsB!Wo#O(=)n?GYAcB_LuP zkgn#URBN+ruO(I`FbLtpsMgBa;Sk1=Z7c9T7XmaQEiag2(zk2y#2Zyg&Hke?lGt-(tG2B4E#yH7peXGpL2IIa4srB0U(M6>pUXb4IacgNnyno%Sxw?tJRzFq{0kA zWSt@auRv_dj;?p5#X#ur8Oit^BXjxJQq)C*M2z9EUYZ(~4ja*|Maim<7>5+JPqy|F z+})7~b)IyCv0G4H`rv4Zj^%*R56mEGAFK&W$qUP7OptO>3at9+cO>VgVxCr3q!zkO z%Ddw#Fn|O~2nhjN>qm64sLhHh8JB8x0y5f`EpFnn)ykK+mg;|T##ybf?M<2B}ZzMOj@ zLY;AKGdaECU4hE*I!FqPpg8744u?78wdZpguPsk!ykgI2yy8x3yh0v6yxO&RwVMyS zuURT`#d-FDd#n(YkHgOYM@7$>Y;CckJEN5jE3RO^h8g^MFZuAP-BX{o;<~ zq)rds3trG%AnF+*2yvc#B4muE?;!O@63OE%36hhO7dGL}1Q1>36)Wd7VW+NzffY-( zVxo1XsC~BH>;j26Z{phAI|FN3Bwh z)(sh(8YrL+p4a&vHP*RE*;PHuxLjV3Hv0EHjKNn{We>qB66YXjyM^QKR ze5nI+(t;YFpZ0DS`W`?@Uxxkjn(z{VzefD>n%MOdiq#@%F$ zjZgsc0DkE~Xn_Wcq*jU;@?$k;O}s+5bZ5lUjYyDyc5W-yrs5%gpk;b-HJCrv7Rj3X zU7^%`P1N^v2EfXg!{BjvZFw70%kqDK-hs=C|HN#%oZZeO*RxrmIY0i7k8_< z?77g1~?N&;y~84+ZE)$T**(7e)~#7$RX#2~o#Z5U?+htT5w zv_a-oTTOcsWAjv6b$_m7)#uVi`PgkbB zqui3%zVH$l!SD@E(U)5aMU5zA5s!3-El6UtOk^lXlJOOOk0Au(hbL z5aPHp;aQg7lY3gqaF5A&F)QWwT#BgWqE5v1*Aa`AAcX!1Cf9x@y78SUxV9Ef2U`_6 zbEqg``_dqr&EDw9yifhmIOZy7Rk1REPX6oU{qKLCoc`tH0X>2_L|H1*l*ctQ>+L!Wjmn_B=OL@H-0i*%tIeE56N7@g*Vl-q6L{l^bLWq>9@iv68O)c{;x=?Hh|W|~NqN`RiJ;bi7eH!F*P%OhwTQ7P-jvJzlDM z$Jhow7}S0MN704p+xmM*Cb+gE*_eVLqtvLEvIb>C_W=Q*QzsjEaYmSTR4Qys9Zb9qK)rzS*jm@w4LELTpg z2CyQPObsyL*py;LWKIjY6cv~*Xh9QRc}f_TG`Ulcg3k_`cSe#^9p3CtTRX+4T9%?- z49joSKAmJ#n$hj0tF>fF(sfS5sk<0gC%MqxA7}@m?4$yY=FY;*LWmiQG%W?FbV>NfYuWC=zJe|-?ExW64 zUxXqB>`rZkV%;=VQB8d#C8&+sqXZWKTg_OSI+bR?l=6^cHD&e3)weIs2%oa4`O8bv zkTJ(VB^DTCZ&N;C!9y1ud$X6O8Ye^d_CHj|)sugo{D)KR;Lo?}#3|Jc5hX(O6K-6$ zEz@>;CDQAHB~x9E=N1)X!t$yNAZ%R;2)g1aZWU;A$^K!YCv=fl%>@%$u{;>CY_ef< zQ6Xt8DGOtA0@gxPoV5}H_zv(wFU^abun$}U5(=Zvo%?9Sb4i;_)@I22m^eGFBD1{Fd_v8y*NDMoe`X|VHH^f}jC zvKz|{eHVGsh(a*20YIC#AOU4-9W{j30`1qyS;C~GNL3V34z2!1>nq>2v!K)5F}aBe8HMj?iEd8jeKP_fRHuA z%m$q+37c0iEi)yMSdZ8H++v!dm)XBp^$_xG@ZcdM{1huz(UewnRAlxWYAa{yh!+?z zClV$uXVYo|qTrBL;tUwfGmH^nDOFSQRz!MQF(4a6WvHiy7-jZt$She((i)`khM9(Z zg3*>)CEbl0(VHgmKWUp8! z&cyRfF-9%DXv+;8b}$V=s#F^*`V|A5L@D&Sq)y*Z=f}9{upZnSk;eJ+I;Gp1hnwUJ zOzx3j-ohKA+zu99wi-Pq^@>(^3s;!vD3(MbME5--7|Zc--If-Xgtu@VckqxK)S3|c5AP;muaUuV|*V^x*B zw^sV*W#ls|5lzy>C4kw>L?xPc#yJj~c3~-GpmC9}aWG#xiv|roQB)VITT120=`2$R zy0iFfd)wz;+?Wc7(mOWSmKyh~HvK57&DqFr5GM90{e4v3_Mc(Ay;JKX5v>JlC{df@{8FsFUAlh|A{rr1G)m;?ju zzGHNV9$xJ|{1w#g(2h4-jJ;&)P0=Kwr&A~sNb|K0(yNxdOeX~`t2Nqa&TQ+W3A<^N zkq?cI2?>=Faid*#Pto z?m;(pZw#i;yN!(u?$|AWnzVW4E^?D%wXz)u725;SDv>VSBseSUg8f0g?TqkY3$1%m zl8bzzll86hM$6qMf*5TIG>?<)w9p2vb;(~`U1}zMl8YSK9^M-#xS=JjSGhqlR{Ajc zmUc5*E3YwAs+DgxnkfdU#f}N|sC#DWaA;Av~8A&9bIe zw_X4OBu?#1as|;i7Vc!(MreX=t*d-3vn#f zEF`eZt??9Hn#!}FC~GvqFGXBLJl-gVYD?dJt+oH`h^Smo( zx}830ev8|tT^aS%%D$4>8eq*J#1XA4v7!}EzyiqUj(2Tkt5ld+inuKbq6i2}db0GM z?3Y#}*L`;&oz;a? zM8-)8za!s)58a`J=G80{50X?5!dksh`iq;cx#mXltfLNSx=OrMQ<;S7h+B$4?o93V z2bN{(UmwQ~n7%!|=ZhS(`?)lxNu6ACl9r+GD$za@(k{^ zU^y#k1&&XR;K}ENZZx&E6}BD-cNw6Q*-Yt3PVG#)rU2$dwF1oMjhrq3hB_tR=o0Yu ze6e}i1NMP9AqZ^)xX#-go^oXyOhJ+q$b(I%^vjolbAPfyrA@=|BaLtSK3110@?; z8U^0@K-bfpRT_`V_ypdW_x`n{0T6Qc(rTHr@eEfmh*?{-x)OM#Ht`zp{IS##0_u0j zI3C5s?8r{ZH8VAH+)lTHz@lj@j-9fK@=P{o=}rIn9Zht^SvY1`o%8UY-nwNI;l^90 zn*35H_}LA}w>LyPtoq!pmvi!tCE}iyxBy;#C}=LZ$iJjKi}eVZrP4Tspj2HuK4GL6 z(lT^C-77r+8v9Gp)PbQz$?mzRcRiq|9rcFnUY0P#4cvG^YzTv1BJra*((BvU`nmRuG$t z|F}Q?dH2UV1M|??A$T&!Pp-`?+}t58kX-Y=F*3PW@v34b%-QY?h)+o)@%7wG^8|39 zU|ci%U~6qU3IJ%@58RW5(Y=(YofEz)uoMF9?TOC8GmeANBalF(CSe~E#!}t*Umv>} zbvr@;#9>mj*0Xs~BlFQ2%gB#4E7wHHg(fQ`Hijn=x#X$NUD4jU7)@ZTm;!FPS09vY z3}{zd>DOH%ZE!*uis&V0htZW-F<7)E!^XPR)OGRpWxO6WB8-u2SPJ#L_jMwgBEjLa zG=Njsuu*HRCwFWeZyY2#@LHD&He@m5pjF;%1j#U@Z~Gd%IPr=qU*%pyrL1snpcYMB zp&<7{h;=~gp)&WH4%?nYjL@t-SFy(t;pX=2? zzNHx-vx?jvqOyCLpOLqsQvbYy43d%SQe7_vlW#=@&wrFj!tU=dMcTi|!LNo{UVyYQ zvK>uZY%Hand@jjlPDH6kcZ)5c8;Q*`?uEu1K9XWj`}Bm1vNeNyYpV5u2U2}DFQfzZ zQ-~n3X(3R|k5+6E5op!ER;*kAI$RP}{;Ih~o>6#hxvs~B-G$e2sR znDw03MUz!6QhL&lHNU4B!+b`X+nR4;z9E0ca|bg(eJrl<{xkWTK>7kglY&Qs`km|JMD2f+tj5h6H0^lj_TNeET5k=jElA_vXYu#W| zF|qvk7~^iVO+9v{Dpjg-W56oW?g*sAhbvo~OEN6N?bJ)RXURA)8TSG_u9>KFz3vUl z*X|>#dT%fVZ1ePPXc7bC8aY?vq!9y*X5jcQ=d3Jo?q=(L&K32VR4|-rc+(lqU4IPc znCv(mp-agX!)+@oTCp>%hd9qn*_7x*XeClU*Aax*jx+QO!6Y;ugbTtPMb-@O{+y`z zZ^Q7C=$Yt7Gxo<(Zofu7k(38INkbKu%+*uZ3()Ni09p%bGXg-yTzpVo`4;%8n3FH# z%1AmC`3jjPCJl&+@I1-t6yt}Gg(_YZ!{@gc(ygSC=20U9ntT@V2f-9iwDVV0Jt=M7%3_%X|>TqFsjnhiQ-!stBxgE3R#gAO_n^j5Sh7!(+=cM6kqxWEP3vn%S=DurOR8=%~Ti5T#5&h z3i7}p5J}0YkXayOD&{!7hornjtcp^sxQUWCYhq^r-mLbPL~74)R_AJZn@TfZz91DX z7pzirNM5gI%nNF|-x_11OLrdI+MTUlY+rbjkavb6WFUbIITvMOLOe~WP%Ip$X8v1! zHr9D1y@z&OQBNHH@4kB*Uu^BTd1h!fA4Hj@4?GQnyh(DR2%Y+R5@FgMRSDK_ynJ}z z>4?FA+uMU+R9`g+M2!OBDa$Lhxl6yCb^_E+$t#YKd_Wt@v#e%3Z}1(P2?%+C)V?x? ze#t5X4Fk{zQ7wt&t2(P_&O|M?5?i3VFKvUp|L({J^ZG$cP#%?$0+@;7;N!W*~q~4b&Vzp9&tfe<6 zH21TvDAi$K!a({f76!1yeIP@*<6$t5v`3-!b)1H2656rMe!DI|>`azR4zRoVxbPcXc{@p=ZSJ!hom>0d%tu~8yV(0H+pyYSHE-{%0>8&@_+-`lZh z=r6U9!7(o2n!O^sK>t%2kEfUt4OkZ+&7gvUBwl!o!Hx3#Fr zg9aGadddwio|9L(tV`Au1?s#ew1y+K7V-+Q=H+vvpLhw6G+}nllwR8VUJc%tMcGTc zHqnrlaTt4PLCXr%t@y=hA2F=Z`6AzSbzaU*LQmTd&ZWJ{uVh;cIduxTEAUHOmxY|C z?EYM)^ywKiAtqb5-A^jX(bJ_(MQIxXL2XZt$cZSSMG4JG z#tg8S<}hp23IaOY$6|YA@8<6!9VNOPYA6XUN+9EfLij`7q6z|8?DOdV;Q-c z>c3jkaL30S*r=_$gM}bQpZA{u9bJFZ=kQAmnBSfd)=Y3kZ7oyr>K!Wv!wox<6yDsi z|ME(GAtZ-VrnDjynMqP-)aN%s>+h7je0BBi)r*UpS1+HFzv?$|S9qS18_}4HJqnLa z`O^hOY}?L|2@{$(brhlU0L{r|&MRBx+AsShnaFkch@z^k)e8>cJ}lQSQ9rld;vve& z6&7{1O&>HgL}^o|1ZupJU~AACU?OxrHQ(cr6f_j)!rEzwD6v?cE;T=$9#0`B<9K>( z1hLEnOL~dwv#Du}TfeEj@TP=(ZIWC45*y!$*R)}lO$p1MH-vZ#*sbcB+gKg8x#0I2 z%W-F0bkm&ZDRwreQ3Kz--n|{T8%MN4m-y0cNW^M|)o7L$@+=PE(sGX7`Vi)!NP_|>S&(YnNzG!QO3Xyh;4U3t>RrrH!P;sOy&*E;+t+~6+1dEbQZ1tHr&lsS0o3nPDHU}cz6drFFu-}8(ue1^a!&le7gmW$1$ z@#@-!{5R^a2IpdsFzk=FU*ACCUho(1tmvqQc$NeqlgR{3N+}me>KX=3HvSs{qL;$r{K_2pp3>~J> zoBhbD$&wZqb+vrSC4ex-3n&{;Iyo3vDOh_W2v-B`U8oI>7k|6BBD7YcSLrg1J4yyk zAFnaTd2|o;b363UP&h>I_8UIm@?!-o@3h*&kL9PSTQ=H{<@aBGgi!n)Pa4?K|6P1n zJm(dsl_*E~4O&lG6G|;BvBDh&W=tN$t#}K@yvz9}!6q*hbRZb>%Uo$CA>ihA{PP*f z*@JEqH_wy*&}K!b_L2Shd35>mrc|M+_^NG1iO6%7go@_IP05e(Y>)q#cPypY)&E)3 z42SRSKFoaW;g4@!>oL>NW*XjFdr2(=(OxvyJ*xmzK&!t>noBirZGXwJc5)t*lPW2^ z{=L2Dz+`pqiYwU88=Q{UHV4Mnq&+(UVO7UlwDkRUe%$t7!EfX>mk^p0sz(nubE(V1 zg5-9klulm!K-08*j%V;(OHySpVce8rpt<#8pH%BY*E+g+aRpQ)`L{nk0ZHz&XMcL) zo&1lRcQV?N^ijQwJBu^pV~blc#DGnqq^xTUuT{KMZ*o|C0fvw^r$)S-_81%cE02ve z>21iEq8Fa~5Ttu=GC9ojz|a1zHHyDEJj_dVKm2Vkia!5wsHqcm*bV;X4)V5!0^i1K zZ#XwpMW;1^Ul@~wUcSBl;q}EAuU^BrVRVbfB3Ui9RK88zWBjfi+F!wG%VV8(dB)(l9=Tx)O4n1jtLl``2S?yQ_D%+>ccS~T zEt@a;poro)T<@bALdc*fHT=)Bt#k&P0W|8vMlx|Ubmdgj8eZS~!yD?tOac7LPRtW1 zjEfGnk93od2Zk%8kZcYmq0*eXI(*H|Qa%J{-g&Ia!T7di+usGVM7@L3tM~0k=L_=; zyJoiL3`f+QCos^z!A_8&j0kgDA&Cjxnv}S8gQi^{7RA?L%F?XZ!9*b0EnVJ=myJfI6 zu)=V-J(g%%(E`iBlTqpzZO==bwL0pg>s4629sZA$Zpx zpFW+E|9$bs7t&93xm<~eRoEh!5SOq#Bmd+2>$eR~s{YkWSR>J^HDZTzN?tnQcdo6E zR?A?8C4ftw&mCx6E)8cefFpddGzG}j8q0V9Ngm88Xcsn^HyCCK@3AKdbm5F-{Em^i zgwo^wToQw!Tr#OdevMv@FKSkoWL^-W{W>EfYTC9vM0zuuVn({UmI)+BUJ%#}5l+6= zJFhHNoCXab(=Z;IiF^UG7VwVOF0JX_I=(%&$eD|KkX=*nW|c`P0i_wohfc>0FK*efs!wB zR-!#7u4okP)NF$x{0e1gb;hV&F9}^Npt{I?%q-=KQDpq52E%a4$knSi5CaY(u@c29 zmSb{b945G0fQ=YDCFGS96(IU4%6fvN1qO#;lF2L+i62e?uSBeLO_4k$@DN!tnwsj< zN-Qc0mQD$Ip*G{3C$x$xW~iP{Sz0T(TIYx^84+d55FUR{5{IBZgCe&VoCI98aK@{W zmTMGV%-ktvh4l#{M;Ka#)fyrD$}jGd(FN^bon4QpFWKQw_$Y{Yo106HDQX8in{#s| zfVS*2aWb)O+s=-;W81cEI}_Ws?TM3#ZQFM8>YRJut@{13R`)09s%{A9+x}|?zo2hq z8kk^K&O{PU?U|8^2CGW8_R3Pt*4%o;DE-^yC9{NQRidxZ?rYQ%Ok$j_vh ze-a$Q+r$&Y{zABhmP3hztnt&XWV)ek-9aF<)%0Do>QtDf*F_)d$c zoNktgpCF*N%H~WDs-E+y)J%)0T<{t6Ob4r6@<~oj<5#-bX|_3sxAf{>P1{zw$JOj~ zs6x!tSb5m7LTWGGZMGzyq&E4~TjfGkop|m><%XA^Ve2nB!%6A>I3v39x2<9p5w6yB zn(J(G>Gkl`X@+QPPC;kJu}QZq(?F>xOT@+&SF4EuTjH}?6E<|Q+VAd+%jK>b&CJQr z-yBP(cta(TKPwpaRedN!^zloxk{n6om+q(HjXPL-%J_V{CV9>mTu3~quBMH~ zcP|lppx}?vST`HFu?wUsNp3W>Aq36F`qR)w<4TbRs7OvEE6FHau_kXyuFLNeohQ=~ ziloykP#YD%24Ise% z_D%7`;w7psW8yV2EYR>n@`j+pv4s}Fb)di0v^5f;f*P^| zsuV?YcB?13nUeI7HWk^n(s^jxrCdgM6}1jNjbugDiL9*AH1$K&DWj~>bmfQCWtbhD zhrLDwzzm}aOwix9gLqQhyhHKM9(>A=sI zDJk`4nT-WS$*e^hxn(8XwFx8d!t4E9K7TPU?*m5G2`UiwT1v>7mpgk)u$iDqkk`86 z;n)LLH2NbY8=33TJbd|-i#!*T`5Ty=>7@XT!G&Z|62bSzL$)Dx(Y&9d89n0kpSsn1 z6z$*p=T61!1D;O(xx1TpRV`FHg<=CR1|*DGcD}sgpOMR_ZR)ukna?FWG+A76r^bUh zv(l%@p~ttjQNbg;p)TJ{OqpObCV~PWZi_wZn_oHHU(KrMHOzN4BS0*Vj7M&+1L!Gd zMMI{4*qQqh+$fRoq;3z5K*|rByVa#Ikc%4Vk$gZ<-VOEbDWZt3)9Gw@p>JteHzF%t=t*LLc z<3n<=QTH(%e_}4^e#{|m%XiM>$}HimVF>GDqFnP{(`Y(ol~t|dUehSX)f8`vzqr+Q zHb&YNSB%9ZGOzczmq#ar(>T+z-ksEezBA43SfeP0N*kgzF1~ExfVs)o8q=bsruI0=pOj8s)XFK~ zxEpUxlDwFc17xKT!e)NmYvcy&awi?Z&b2u+=91AJBORA{t?*kS-^P_uJ5h9UQg*~+ zL|w-`nA&aPP*PwJ1|^+B%(paB_J2Dt<$idc_3Slsi$v=YWUiWx-4n}%nk;Pxn1sUf zI#)0)De!Z`W~TLX_XnG5g(>E-+~U5I_ufwOKt@01ot74doO^GEQFHNGpRcF#8X+WO zXWEf%Kd&mJ3pbTN7|65N4Cj^Qw7zPwxzbbTGGFjryff9kY?;OLb-7^=I2-ChRZ%H# z&iDG1H~v2lhmVYulsf_$pC3E^{`Y>cgnsVtX$yH%gna`1{M>KfRh<4k9bQlFe(w)2 zo;{s|eSUzq+@5!DvV?_&zR!!Xu+~S=Ty<1MIS1)H5%5DQC$o0z9hVI!0cO*1jv}?0!M5_9f68N`swEbB z@T$Zmo`}WUIU|%hi<`2UN?It1JV|bHM*LrESbf`Wt7gwGHw}cKrc~rwW^!Sn>Lcv> zWnow4KiVZA33cI^T8cH*42g?#w&U)!cmyYXBprJR|IbzAygzEs#u7Hz@Cc1i<7@jb zSDvXYwHCd*OGFdub0sB?6sJ$+UL(CrUn0~K zrvky(9yjyG|D2I?0t%sZgPOK{5{hF942mEsK^3EvkpB{pLt1?8+(;$XhH%zF_J$rZX#{LCTYuAL69G-^T}HV^CFg_CX`+NB4vWh z5$od0=0uZEWhy!t>AUG>qfc61KjpMmo&LQNzIM%#2GEslB!N3Ly#^_5r31;J?h_ogba@r zMjhAYw7;aeQ_ zlyIxLaJS`HJKUq|#%RPjbg}dFZx7Y6DQRYhdN102{NHN8!TwTPCH!U;2lQ7_S2dx< znO_PBDWgwaNUXbl_osN+rFfi%?{wuMpW*Pmj`B_}_LZlH8p%lt)xjh(eir2|nv7`b znHIB=oL;?=rBqz4SI^0}L`b$M%^0&>u`qNUuu!>CGoreuW*2u;b^0&0`7hBPc@ylb z4l?hy-Tb_@lP?Iroc9Yt+w{M#%Snl}` zq&Y9rD5VNtWhIu$piJgvik*^YJX{*b*j0Z}j0EydyM8p}G#YO;D8?--`Y>2TY)p4> zN$c}LhBgcn?PbA3N-dxx9!mJ5(6Lk}2I{cCR&lGeEsdE7v%0;xCaaa+*wUJ9G+958C{@47 zOY4T+QlxY(09Spo_nigdQNDf;9+~MgOs6yJg<)LaR&7~EoYoqHf+X+X5&d+ zw4Ezfi1Dr1cGidCeTF-`&LPrHCb&no1ffTtk3R>(%Q{t98epE z(%c=w+pqQkQWXVSHZP^x5J)Ni{TMDFGY-@f4kWgtwcupQ&O8a@GS8$vE8Gp0 zQc0Oba~Xb_uW*&g-(uRTV&kTw6JR*oHJcRZcQz%+*>Mzj7c;?2m=VpNZc8vt`m*5a z{O=``A3$QhIR*JLA2W`&^Y0g;+=_yq2hqdE%+I>!-baOWqb+?Rw8X|(%=MVe)oXEF zdG)35kx@K!SL|e0+;l5Jpy=6D1Ho8U98uTGAXXh1-nd6BcRKS1x0#NJM#2@|33Qb4 z^#1C-vo|2&Zk;;XQ|J0Voh+aoJmB>5kl7$}JNs~o?*KZAKVxj*av!>J8HtIej4N|) z_y{AUDU6f_@i|onDb13HrLTD$?+M6xFi9B0TpG_}g2y@Ym}w4Yr=rk3dzTJh>_v`- z$XD+sv%hS)8Wv!W9$YA%mven_5r?|uXA5q7MT+lM2zZx!+2h?ZXFFp&83z7v_jojh z)>y_k_VWqKpf;t$V)y%AeRs8Yl1YsdIj5VUL<_s$+B`aVS6Erjjkz>eu2w=X4*0t4 zf1T%ecsoMVU7$}{EY05_ z@=g!qy!vZ7%V>EU^cvN6{O%jaZ(Yr9kqRn(z%o(F4`ywSku!|0`Df9o=;_OxPgB1a zkB~KBCSyTj-Q(92|69-m2(FKabEhEb;P}%LroG&f^skG#qsdyR^_13I2zz&V>r1$S z@&)&UaWJ?+0v67xH*^}=C1)o+*I%?uT>Sa zWYGknNO&IV;O9v_bYt-}XgUj0O)9p$yaG{V7yV*{!wnM|F(`R7fS->jU_pIr%gUlv znoX?L7?ys7Dvx|377vx5CNH9XFx_Zbzt>6|Sd`z3TUK++EcL(;5^-cGMs%f=yI9oQ zvQAwjuUj5Nmw0^^BpS-Bi<%xn%Cz{7(uL_N=jkEYIZRYN7tJwc>wCyMNq{ZvB(AWN z&IFPRr>EM@ty$-@H5A*_xw?zMUX75sbpdNx#K}o%y~;#$K~UWr`>e-jYr-9Hr5_=Y z;2EZ5@{M6+gnkt^De*~agaw%dl^#~2(W=pF($~~MFZpOuRYZmri7?p{im7nLEDgl` ziV##A40p0u^9qt7fY`jy5;X;KiNSxwken|Jp!6k1_Qsm7mi>(+&b(F*UI zcFr5;t9z7B*2!41 z-Rh|7ex>8yV>lv7abX(`SFk><}xNR5I{t7|T1lJ$<)S zvH4Jd8btCMXMrTv>>?}3HtPksN6urV4b+ju%^MczhSQk#<(PgBO`fkY$}`(WT(|;y zSgK8hM?r5}3E{D@-IjlUHE?X`75zHcJrS@?+YtgeI_N)?h;iYI@R3g$brY3uYDPu? z6!LC>jEn{k-_Wko;Pkm_*T1FnxZ-EP==}I$+@CQ;x`ia0Vh4P)lQ#@!66-)Rb-|bL z9?NErUL;(7zL@dil2ehqKgiEGlZ;VzYc$?>9;XQ8O>e=C$ZeFPN`^`jfn`gLo4mAx z_*P*XW-d+1m(37eX4gbn`HFwjRR8qI>QFxwDBMSTPp~Rva^%U1Db;CQlVvc3d?ki8 zYc)+_k`%pxx=ildIp!u`6r!A{Qq1^kZ_-mlj+u%*Oelx&q>Hf#FFg)FQDB>UK8Xmo zB~0Il@JG~wJ!X|mAN)wnkig$V=-N5c8IJ(z|pNUME z6#Ca$UR5G+U*CgYE7V8=0ZGWUuceKYB`nt{|BQARY6~(F2pmI!Xwwq&gj~g# z@+G-Mj`@?P-|PkGz9wDvR!V!htP~Q9#vf7FECoV{ zcNDu{%J3_!h9N1IR%ds5!Sl%Nlk8b+a zd=DHv3rbvJjM|VLAQIN)-OqUiEM|^5M)T5*5+*yqg{h1DBgWQtqjuqXM;s{^am)g9 z6cacdk2V3P=v`oDVP7>s0xwLr0%JTBIJ48!7x|$s(Nb-` zAX=SGdagqnu6FUDNnv(ETM1nSRz>qxT`Nk>9CoU6KOB-eo~TY&ah{m75G0S1utUqO zb?DS7?N9qVRmLJ$uKw!k!AxT|U;Ns8l*p;sJg;T=x8mHtNVB$?B;jI|rb0!K#!xAB zKJ}ue&&G>Vjn$#P=E;jC99Sry=GKvX@k+*!Mi270x-;%?`L&2Ikd`jO@; z7Uvxtt4TIyDjYMZDYVg|*)Mj*zvel|sfT7#&cr)Q*tGOV05>EQMOGT-Ke#*&gQO2v z9E7YeYzlVj?wVUA#gu z-#v9Ry+uCx>?<_~8ysdA>goht>L1Mx(hwX?0>)PMX>k0D+O2={T-wIvz3m5?FTbRE zdT&g&8Eor{d`)%K@`JV-TxZY_x_y(tdmY5OQvUp|99rvF{H`?wigV|`WzH5`8N6au zJS3-Jqot1?G~WDKQUha3+`Gi6R>_JhQ7-&t%%a8&+!VH1-++W{(m!NqL!EX?kf%|r z^*}fqAS!SbtI6Gdir_db)?(31jG5M;7>`nm6sHHvkxnlANS;3TQEyl*nBBQjql7Z| zsLKK6rr2r}@y?d!GD>^g76eKTt5sjuA(VW%cI8>iaxBG7-?SI->Zg9aJXfb)<6jjZ z(#B;?<@Wb~n%z<-BjyHk#OoWlb$!V4H?edRS0l3D`~U3Ne;?hw+xwFPwbbtUsFh!L zijgX6);db7YzS{W?hnV4W`15JK40WtZ<5UY`~C9(Ukgn$Kd+4PKeuHwKd*=J@&d{) z{9m8)Gk_1wdlN!NMBSEZi#JC3-1+}q~E2Wt0aUti)y+cJsDd^ojMsMWZ1 z&IdeX;a{sORB&Zv(;N2}`O-XrEC4h5)BAM>nW% z(&Di5ygzCpYfUM-HprsDH$O4I2z=k}s!rr1yizD*%I>@BN%)G+hjvsogXOXnXD&KW z5%kvlVLD|wlY9o#RUG(Iu!9tG=C!@cQ8Bn6+Czhb$}jvPpCH|6LW>(g4V$X3qwDvv z=6Z1vCo|uy-xXBM>SOT$CZ_vO2w+(p;YXU-w9faexkN{wSuFQA?HUK@Dz9*{-^dJj zT*Ds1b7)a)B|SCyQ(6y=r2=Gvj24^rZ)~QY78j>&-J?=AlFz_>J!g&n(PR13dcIi@ zzSb{7VbnB<({-ELYCvm-_Y%Q_i$hVZH3L^Z98F%vF3N1CxE{!{edAtNYhO*@AWyV( zZ*ELrRVoON)z4>1^{7-GKry)>eB15h!+~Gem|NpPjw_iWEi?YxfFFq@uy7|?=lBpp zx8V{3AEbBd1iU;U6_vf-QF+&C>Ut^*06#SVdHkEdmttD;UrmTVxl&E7qjBXgh@fH} ziv8RaU2?xxH<_^ojd0N2qMoy>S#j-nyN$M;b$&Tzw%pl&2yygab44c_G8hV$F8+(0 zJogFRGvlU9g44VpIR4-APCqv9YbOho3sZRReQ&&k)x@*s#*sClDg~%%iSV5y4Z9r ziInX+y=dffjjD)Z0QYyH`hfWJ*DIX}?cWnDi__ceaI(xWcTv;MEJa6G*gWC7V`*%* zows>6@n+3F{)?RHZfYt!9=!$uulsKIPTNz_P{o3&C119S)XZCV3l#FQ9igc<3X?)q zXB!=!>%#2rA#lcCMhnBqo8RFj_b1M;pC=m&x%d~%Mcd43-E?A#^^cHm!WNeL4NfFU zi5M{^KEt$4cQ{=!W=v#aonI}v(vX88P$j5+?xV2%_WBJeR4dB&A=Ow@rLcTgMKR@!A{iA zBZFK6X%WD%8VZoeZiTpL8r#$<5h!0)M>DVw5+rv3!-#GM393%I_CiD(0Jf^4x z3Wukh6)I+sQCn!PiS%%*x4|oStXn9N)||waQfPYCnYFH^^&A+PKny*Kg(bYJAn-E; zdZ1~k0~PP{_Q9~??6hY#IRZr5fsC8_BK1LXhG3-dq_5@Yd5X!3W+38b=u1LT(zT1k z$N_Uucg!k@8`vp)im_dx7DL(vop+b5;Sa;HI%RKjSnVrXMr9P2*_j~5`0=Mn-0)iG$XZW_Dn~&vx?=ThwAKxcu8NAT4p8Tf=I4aR zEud;H_M#hzarriezt7E{>wa^D*5K4;kC982o1o?bAsbj&1*+Bo`=iSX^o!X9563vej>8GdY{?VqXegdOC?Y)U)XaiR#`@{qt1D4Topd~(E#e*X1`*>~vp%WQLW z+MT72{YJ-4{6aM*9f(Ec1nH|07E>}HSvbP`KpYbSZS=_@aDp|pGSpR<62`b~)f{I_{|SqR2FM>>(P{hVSQ1dMDnemFGqW^S{v*o* zFdyd`UwyM*${S9U<`^0!EtN6J@bDN_(xX{M?Re*rXaA*2&j;=@lliA(k3AwVK@!y( z*5`Q-j2iw4ALCHzurTB_qdk|q>TcQdIaQFZAWWPEnTkLI>dN15*5kKzVg%g*wRK~u zqpi?>Vdqj_E#~Duygsw$oGLPJU{0Bx?5qoG@UKxEAtZQn1sWjy#}oifn8TIV-HZY0 zm$!h=dGp&uNpF_I0fxH?l}^UB4ubnUWEgkrFl5ULu@+F)F(DFr>JbQemU#^~Y0zA$ zks(0&<9dI1l9Y6lC-ePv({7X{+48j+qS^(eLrTCuXSJXZB&yCk_zVd;YL z=$^O?qbfCqCyHbX)%&qSHcF{+Cp6s0%*Ldg>mTHEsbmC4Ofq@tnUndme<;}Dr5GtJ{V}SDF+wg@!nCP~L6+<*>^A)?dL<4z{&+#8zw}(=a@5RP zD%8?&AX*}PcN<}g?5)w&=MGCM1mP8#r`!K%;qa;`M>Df&xNS1yqi`hXw%IC~JwZrv zST6{?3ym2xB6C|CeKAiW*R00Xhv~TH*ZO|^#qTJzF88@ymkND8$aCffBm=#O9ncN@!tP!pBTcq zfK(dateSe(%Q`Fx^G=gl#x8gE!gYq>Q9EiV+8=fn*8R3_gbo-ysnDv3(DJDAJhO#{ z-yw)Gubpr_J|{J|)g?x>%}gx~%p86^U|xlO@U-Yc@sx*uC)kY=$LdNZ&lk`4lkOck@*DY!BV(?r6_rqmMggg6S zZS>{5#>jzrOvw%ev-??gr&x^k>Lhff2~B9mNFi>%$|IxY1UvhrH+uH}9yo9Gsx-4$ z;U)0XEug8?PXg@O)c7QtlTHpT>#MerVW66Dl4N%wZ^ro=NY7i=J=LuCww4cCA4h3c z+Y(|UeL!Af`APmDm#fY*1qqtZPDJ{4-B2s2-M2)~KV1O(TQcfIq+`&=A$jmM7uk6HS7;R#OuHBW;cNd1EB~5e;eEkr%HJwtz49vluv)GE;9$^>1)k=!WSu2 z+n6>M0^jcM{s4~6`e{}Qa?NNhT(IVmno=3q_G|g5W+8a7DA;VnQ&REzf1x)nJ3jV{rep@~{_* zO}Z&>^s1O=+2;b(U6m5y?fZs>&4QE5tleWa2G&{^>NMTwQevnUSKA={iIg5dp$aB@ zf|-%_Sx@!43DhMSJx@I#s#Kb!pMJ0+nShB)QSNK(MGo!`5<}kp_f2jh%Mi=6Xqq{G(;zhYW|R9o@YhJq4b4 z!6Ep=A?&mh1P}J;`kKgZT`+TP_V}50Q!{X7!&f1jh{y!3vZ$MZVKOh9iS03>N97qJ zt(*KjsS^x$N!(p0_<#Rg_I$W+wze>>=RK$!P;06yRbh{Nwv6icB~`6TSKEkNxtC*~iBj?z&=*a^EhA zJFMUrzpDiRnAXvrU?#|{&i0qr|N84PweH)Cc7R{I>j zHCkU_^jQmZa>dd41IG7z={_EPU${w#-hEGkkb5jVCSz6V`Qj=nmTjT?%^ zL#^?-KGM5~wKONbH4frf2@R2$zIFC1uJO>#kOnAHP{muVgj)3scbS3$JDi^4sFjuH zdp!<0;{lB-?57&1q%_?KVcJTjDi|7{0wYpQKJcUqy7r?YHuPZ=lG=ED*eG8)Xc;C)4F2(3!PkpscHVXp; zu9_5fZLu(lbaX9cSV*R20=xJXYM(lrcXJBj9XSHWl=|2QeY$?_*i?Os`<|TV-P?N_ zlZd!VU_(m_wK=RDM$>!xaS$RqYhbzJe%c5lD^61hFv zfvey^*=(@YGXdD1zQLa0Jp9B{%mj%31Ut!uW*0h>QEQv{zUuoAwqn$PtvFPOzN0N` zEkGixS}$TVB@yKN^bXU@&t8zH4QUBUrCQZPxtlmU%(oNsBRvhKUG~@K)&Mo{smC(8 z1N!h~+OTX8FF`h96{@_yxrW3|nsf@;{;fB;!Z(T3R8uGDvg9%2^0#ZP`9@NbpL|iZ z7i}O>AQ^rs-e?Gcwm5x!KGl@2uaKcsoH2m)()h)uRVjlVL=KxWoN;#d;( zQ=6qD2My?wubU~22Q*1{Xb(4Q)|U7zSO1V+EC{=aC`d{^`AjKB$&}d=v&}9aBLc&fq%mwM#_juKkno4gsYj51=qf(z8dsC}sVHnAS zlPB|p>X~XiHivvq7G$!elfVy^Nww9;?SxuLYPBNcxM>{4oeH3DW%^y<;ODouhic_GQVKuz7A>& zAPx=_s6WEaS8y47MWH9~S4cxp@>2u~-FOH~1%#294`VUsqu*`{k;FmM}-H}BTpAb;;;}oCR zArPVVCvFj54!5oGz9HBi>~YT@@zy6i=pnrM>S})hz(5~X;`&Kw=nD(0;{@$n4F4wm zgbq^vJ^B<^KC0>o8ev~KIR-6NQqy3<4m>5+sz_^xi_1brB~K(1mFBs46quIba;C?d zLYF4QnO*>zT?-i@T?CxqKfrX+ z7zJMVSrUr0)2Uidt!b=wVd;jT=3*eR7OD1DMya|=A!eB_OlYR2Ot#)!-zYZz3pW4R zZ@NZ{p|`qY1hC?x)F-VHV2;`uc(N!Z3i)ZV&}z7wH=4Y`;)Lz9Gti148``Hnf|W( zLK56&?fuI`Z|4ugWW-h>K45U`B#gTR z%!?R|yC}?yFpRqp%!>ewI{@Z|55}Dr=7k%^oeSoL112*J^h?I=(qy0brTiah#HAt1 zTnSmJfK%&gjcc zb+SrlNe@5dEn&AC)P0`7N4w`+1mY!vC(zg%c$B#>W672N*EmAR4gTkgUp;`Zr~B2O zXbG2!i0`4Ch+KTc`5NPJ0Dg-cUXMW$uZzp`?k&vV%jpy{S1Ep#5J($B#s~ttBa{4c z^ypF0%x-s#q1uo0xPF#!cK1TALKRXxgUn@H@C5^NcDLbb6C9;G>$muh83Wo`aZ8KG zm?^WhSV0~SQXdaeSyRl6oVv!j|HkdZnOTZ0ZXN%}lw(0^9wf(9E5MyyyVw7T?k~k` zLw~(3HIlZ)hl@l<+SZ`o3k#;I>L`JY?b1+VmSI03fwaq;i#egQ=+RP+Rc+Yh`B2I( zEXHo9JjBwN%%np_^~>DV;Rsmrc+_TFJBntob*}j08LpBdQ!PNPohP-ALrulBTKC*2 zlqLfp3cnEE-T6|N-6pctoq*&69b!->?dcCrg9Ks?4x!IIEjKXST_3*MPVYIaGs(=r zU(w>am_l%r&AF-xN-bqPy3L_yd^|Aq4N_+A zah}L5ig%Y~vjL6W%sg%w-gZ1^#?A+v`Ijd{&EczW~@JddxsAJcj z!?A6*z)IEc{!@he&8zW?p?d}t$pXlx!)s?xh;;kJ#GFD2Eupc z?mOiU2Bj)WJp3>ZU0auzGIkld$%s!IhFtgx+O@KjL@serS%a$#GOn-gei7PGeXXJ9 zPFKa7y5u`EasIAd%UKa^Yr5_FuDXiUwww*kDS4Xt-!Kbifec00n-KXu331Y zULzBPTiF{&Cu!g5T(EC#JG3ZPH0XZFQPn@XrsN_kySjYq>j z`%Qx)S@)>iU;cM%bqW?ld^hfY(C6$E5%<62xGBDA=lcg&bx;|{rumY0m`{dcS6iLG z@siJ(3Y`=yM+u22hhx7^O4}QZtt6lRQ1TXgjpbb^@t@3iR^{UU8bTZIFD#E~&1Niw)4YhMY+uyd*YB#5~(z*FN4TErIh{Y8l9zHnG1Gg1X=)?QW0hbiz z#r&8~5H1u^LlA-7gRm9BUX|Y-_ifQe5P__aaFK*{8hSbc;)S<@+u-ckz>iXd0qqwk z)PqvgzdqtG;zg0liqjamM-XKPcU^+jG98y2zMUJTDA@MF;N?v#K5%)_x4LG`3~S^< zD869L3~N?TScIFPM1~V3e;1@A;}(hijfeUolTU5n?DfkmzHkNk^5S&lPyWL(8i||V zs9${GOqboC;157ps_G3a_Qnr0NK+B7i3#Bth=G6Fn8uuGK*U>ZQ^~myB=fI_rxV;j zRF3&Un26x?-`Nzmz&PDR`G)VkM+m1)YmC*p3faiAJNd($Yq~Z=_5w>g_$gVjE zEldkk$~eYQ3bffA?9Ak{O5`R9g~C>|79ZaD}+TVQTWdVZ{`n3 zYb>u$^K>z#+6J@N59yqd%)QtQ ze$G0^9ul>ttiM+ED0RP8*Q%tl~GTS)+j8y1} z^UTi7KdPUPz?gU-EZyxzg=L$XBuTY=*OI60zVIure;Xj{`)T1U!@7U@f3j#4Yh`<{ z*+R(gza`%jbvfxTb0#cr<%Lu6>^T(nXoi2{n0h{N>!Y8F;@PJBVsZsdYt)B4-t(-a zp^?`JYa;$?!Ovl$oF`=Ius918B;(DnkWl>oTbe081)EtS3gYl`_O$jmXWklDu07L? zeWL}icH1YNq?=n_=V{v|l_hyh?#F+_auqPS0AH^#7Q@g<;%wkp%avrEcEB^@&)1AH&p!GZ*_8cXZjU3 zd66tOqjPFyQNBIR2BV|4Rx*l@P7EF^V(29or1!MLOW3L-W40w+K#ToqUnbZ`!Ce>q z^z7YMC7Xpb8T$Y>k_m-3VB@{%?OOT4?x5@G57SLJ#K76ei)N@Qaal?Dhw2KPo+VN4 zjr|~yi`AUW|NZ>*C-*#U_UC)&^cH;%1#e2y%i}ZJxBZFwMeqkO>VLgSUv*-Av{M8) zh8|%<2Dcc7*F~qtp-Y@4!3c-fQma)!dU(dOQQjw`&;2^HU>k5 z8<2LpRB51MD;H=wr&-Ycr;J)N=~88W7(CiU3SCNtc#L}b47_w&W_6P0YqcPk@A8jf z8=1ZVb}s6y%pk?Z%6)WItXt+|RrEy`?8TE|vpeB)PwJB{e&9CK#Ib{`MNMbR@9px6 zbNc+o^qS^nlddhnQeH154pM3-U-bp*PLop@y@X4c)U7k~4?d8d1*bwzj%(<1C*a2( zz3EUSqVLI)%`f8?r_mZSlgO3i-!C8A*(mK(zWYN z8xz~MtqD&uv2AnWOq{2_pWnZps{2j%+E}am#jaJgue$J$v!6744D4qwR;C(2PWa8U4Ilbm3AY^DWlc8<0HLSY;? z`EFju9D8$`!DOxMzu4fqiSuGblg~|Mng*ZI!d@b=|MVgs1y3% zD}>`dGJ0WGd4jserbO3Su^Evifs8?)7BCRp1N7H z<6OaT9%!xG8}phJi|j(@@;DCCUfT_6kX<%PhEEXJnUC$>za@rj*+)38s#0VaqX!!J z_Be@ad95;8Osp`ZCM1IEB5}g3$rh~3+sw*2TQX)gg1(3aFTa`AA~L+TS4)WRRh&PO zbb%qWr|d=3+fv@(uu<gy!<2HlcXU#5k*ZP$!!|By6boKq^2omC`TG6bfN?r;dpVK1uM{ z?h<}ePg{SPsi?#(q~QLaPuJi5{fpkM@x(LO8;u~MXV>a~^CfMPjv7XW9|pDC0$K(_ zX}glgms+dGIrlDgq=K~kFrh9mHeA~q9Ihi+ipR#UWNSzZ;4xKEWIz`u>4ZosGM4=0 zhaFmXIRxHDR5)gzlcO8boQHhB*~Xh5yRBzT8EgFv;(k3AgRi#(Bx(4-(TD^KU$7Xh zt~Ebcn&{8a{C29GK9XsVjdpo-db&D{rY&lfnog~zxh!UdDHb$;TE)3qrR>8Xs@9|I zG$U8k+fk0=@-q_45yMxzGU<3q;Wk=8TOGJm*?G3+-!7Qa>F1?_l`z-)Z5x(xc*H7uBatust3!AyUQtw z)slID0mPaG28{gO(k$7FCJtzLMJ0nu)d*!e%`ozb-$Nf}(CB5l_p>t7Temv)$qg#& z^R%W2Ok&s7`dDGG^`H>xFY7ME^GFyd6=}4h$XG_>sNS*|cP;gojKiJzGA>kzb7bpN zvSBj|qpV>g7mj@;dpQdIO8)t6JUNw+6mPsn)Er2J*e%WUN!sfHBaA`-N$xJwYcnBz zmRWkh2&_pCQzLf;_C}|AKph?NzlLnvQzA`1Pq2~diF(Exp9O}3*nV|tWw`BSFKl(U zRVJ$!;}M)aj*{sMpO~9s*}%{CmEJ>{TV#=A3H7b7pS=*mE3(hf6L-i#8w^{U>eMf~ zSi0nEah82g!5LmiA;fCxgkbaVq0}pedv^CmnKMwzd%$SwwsD_$95Fdj^(dGiyJh-> z25~z}%-Myhwf$zKDg`?UyLc0z2_04sHOzL{K9*AyKL*bZggI-=8N=Pu4;8PxIIitW zviY>8O^q5-*hzKkaN<{HDU;=5rmdLMx&0Kx87yxtVz-Nik>qQ)!o;PLlvV{nyC>v> zk-GNc?`0;RN#~SqkE*S@*(I+C3m#d;)<65ML1?wRLOQ)^pna74d|@Rcim30wxH=pf zB+YCnAQsE%6zyc3$GMpjcfUfi*3fLb<57Zrh@rQP*M4Rh!s6&P1MFX0n*Ct~(HW|T zflCpam9}r{htI47!}rFqh3oF~jqC0+i|hWTT;VwFyL2v@7qE2B2m0NkAoRu|0jMM> zK|BJKAYyhEDnmD$rUpv@tWtyh6#I$U=J@VxN93r>SB}z~fJRw?t7+7(`F~R(Qy!L5 zAWY*H;Y!!LvzZ807yoHNH1FU=Yl7|rLIMusa)lj!lNfR%Cd|6kY5a zGew@x6vfC8Z5a`ahGSVvD`f!?Zqjma#1zBre8YDlZ*+0qu#`xY)QHoK2QL6b2Z#=bV3I@9`r*# zdikd#XFPu}uCT7@Ba#(DeS3|>0xNoLPxDLPPJ0yEL2Nr>#SwrH+iEPi*`9zjA}fG*((36fC5mKL=`Xf|T*CaF z-etFUK!2FMYSuoCwZ@6fF=YWk>M_+Y=GNX&rSq^JFSJ){`?#4;7rP*w$hTfF$HYm~ z%4dMmAzn22Ov64-6&|r!Hs`d7xnX=v0z2Ba;Zb5x=gsZ8wJ+B}$pthfcdvD-ubTOu zAA-^={Ej~Tn+adl25ZVyKkNj`-#9HEyZ|UivkTTaOJp`T{n8*`X|bOyxLZ1SN|rUj zPq(Vel5umAu4NqO%)CYSrkOHmIqr2@^E2Ou+5>-oQQYQo8^gQJZd$sVzhfIm zJBXB9oWTYW?`mvfosw&9{M~g=jpm9K*law%5*`kBqv+lpD(;ume( zt|ZHIh+M$J4^iUa+MiKtF+odD$kYgr+1B(BhqD3Q{A6q*6RKdTRvjFcJ@LM9!5NmM zC&b-lp~pe%fhpBfbBDhybZSRt0p8c8d=Y8w5Ya_K(6LzNa&Ejudmr}f{c2aT=TZ*_?gOi z00(c$kg1!*Aa0&=3DCw(B7TO3OTu#LEDkSEd772CwEp&fCc#Ic3MEY_hp=#HUTZK3rNkd(1pWJatIo%{H!!gB^>UQKVGz1BCS#4 z)-g+5g-Swg4NcO^kU83_HbZ9B62>@MY_RYfV9Du7hF)2I1e&nDNt^#?bY1wUev0)Q zDgjnFM?%*i3b?wEyxjdWiyp_86UM_utsVgc^T(%nx3ha_|8`HXa`AHS6lg?ZVo8C?nN-`gYoUkEWjZI$W5I zFt;PvS;gKOWhPnY=AfS}9o~LTT!yo#__NlZZ)+$0F|cr$TPJ<{48XEWe$+hWK?*dF zX9-tREzr_pvxxt3QO21RaKp~je>LIFTY8rBALmIAhjVhJwTfQzExGp<4sWd#0d>D! zES15UzR9F?Y3Cl4UJN>#$chj2VH9r+|7?7*|9ih?~c#wz*9*m)ZF+lX*{a)Sv0<$nTA6quATUmCj#_>N%P{ zX88rVFPiwe(-IINo{*uuFVguQgZB$F47zdA%XyDT`5+YNOGHs~N(%H5bVKh>8wIM& zqXflxm>)tKdzfE_u{nnbcUaqJrb7pNEa(t3BK`b;nA;d!=SE412GSsm`B;Ytn*yEa z34_gv5?4kP0J#WUlIhC2~(&2cOOpcA_#2*%+Y)QMHANF52itSG(s zI{rA4wf7TiF%u^}JYhWtkio$_dG;oYxTm9&>ex#@6PJMNn5}|pD$Ydv4es1|Im`iZ z=ePVJE^p4BR}d#doM`r*myu0BkaG}@eej&GtqDWnTRJ|^38cWy$AA@|=j3O~Ah^N` zIxsc8YifE~)AD0*h#MXA0crW!F$iM6&uIBE++&#DH4Wg#{#w7g(f6~v3VSea)4=81 z|1f@|i_3EYCBQG^^6mRY9r;{^JxCU>nNsH_IdqJz!C78~0X>EQ|MQrMxc^M&gP!33 zIS$m=e`eDLm+(J7!UFDCS}y(n8GQS3|NWvVHNh+}a6o}OUaw%sgS%gR1@}0}gPSO7 z1_#3?a*@v+<*JTI2*T&UYk;%+81W+d@Q053Bwz_+tf!@3cV;f1o2YR5F;hgqnO|wr zfyY!aw~2aY4TG8ml+SG%+_NJd5A_!W?)g8+irpDYWE$Ml$%@^Hz%Pj1xrt{Q+|x9m z|BdT=(EA&S5afVNtj87kpTHOpl+SH-6%ACG|1Y+@idK#Pze@bPIVy~c8z7D;bl#lh ze?t=-_`exCESWU2{VzEu^uMnw=Dy+j9>fVagYvnTUmAT5yuPAb>7unT?!sDl^z9ld zoU4#$8<-kwf92@Aj|7;O?oxKR!QFsl}j1&onS4WnrEn zQo%mSI3dMgN?av1Yq_KQj&d}Uz+-zwDoSfeiI13yr8S*as8N-LH@@XBxt`EZ*EyxT zU$+`)lFDl)(B~5o$vhqrQ0lzpsQTaRMFR%G-}bi9B2L)JVd>qvK?iD>S8B;(sFaHI zC;Vj}YO_I9kkoL@nY?*K4*uJY19LsoP@hQus9o4+?)Q4Xz>_Jgz349@Z-JtU1Eo8m z`774eUF=@4nEGh9b5o6_(O*l~aNQj5E+rku&tj@8Z;t&%fUZ0uw*Ydx9%)EvI91H| zuzw(E+-6%OOjN>z`hBx@#Hylhin2A>P7QfTPSt|gFCMtg!-FYRk(4EkNSq#SQ4M47 zv`9Oq7n_*52eM2)UUFDl0})l2;{?bL^`ug@oBJS2!v484a~#M|Q&?5i$6V2b@mwJZ z*)o^cmsa;amHhF=u$HhRnf!~M-{nV7=eA6LX>{e-6?1UmX%+YP6on$bA)k*=ju$h2 z&3k8WDCmb%Jdm}>JC$emehG(ZxGT&NnxDhbS-mYkFYg%ddO4)d{_yp*A8*B`G-~!i zV4Sc$7nK%HSI6YknkO@%1kL~tnqDKkt9CO{M}<6^<7t5uZi;1C_#GK&EW!DV- z7Zv8Joz^uS*=37u3y#cVv{tB`Hv@OcoS=!Kb%%l{9YWN)pg+M`B<@^xjERj$lI3Vp z!lbLuvcc$IU-xXMF#-f>#DNsC{HH$={J@qZH3KZps+<*tRuY5!Km6vVIf^}~f&?kQ zV~nc|dm=exGZ)3@;eV{*sM_DYF@ZK9Tm!E-$5i@HhFq<4)ZSP6oHiQ~TX`CeO9Lig zs?(uXM@`=UT#lQTq_f{^@ktPEzDz*euKh&cADh7@bY{MaPT+PTCj^|2OGyf<@uDa1 z(^XMR#JoTJ(z05(>m*7vpBleH?!D1f{b8WW?~Qq3S^qsbhX zF!i$%{)Cs>&8J_M2mTF4U~^)EUGu(X(9f(Uq3`EJMvOaw3PXfXMq`lv)rd2&pFWP; z5{!gZX^pU`h8-7RMDGR~6Fg9x7{8+`Pq*5F=rs4~&O_c_Bk~Gt752<(F)Ur&YJB1T664n~d75c!mU1 zR>?hQ<#nvMKV0Eym$V!hgcfMINqf9jk*WJXw&oF8tp+gPEA@;_>a$cq7Obbz2lVl%|1z>x% zQ$6Ox30T1lQiGdmvHu+Q;8hU&$!ek!z9FF!qm__SW>Q9p$;euCHdo~|uI6?&Rv%0( zp^r;e*1lZFgns{p8vTB$Va4?cA8&TLn}20zZHadII=q+*dkaUj=ifDCbN;ip)^w-K z)fNph-M|~FwTPVo{QS|*gmO9#y&NwXDux_i%`t2`)z&9xAT;IECrdKe1(U}A5Jh7B zKO`CKLcv!e{Q9++iyNE_+g2i#rDl>9eYus)_^FbX#QTp_D zXK~2&M-c83{C5EU(>k*dhG$%vUF&`YDnxH$NJg9 zb>9}uUM+@(0?))skVNJ)crgj}*~XD5|7jc@lfoR9;`MLR(F2)Vz_}1VgH+2$V&5HH zl?4o@7MX(|^#sq@)z5i6=SHtA!zP)Kycml)Aj14oW=l5tSN|GGpjJ!4m z*)pa+29&~?ZXfAlfLPb{2Q2SWr*o8nsEi>~VL^DbiUvHjr>cKSm>W-Esnw(^p)X&&(ay!KNkGkdW*Z`!0uu~Ht9nhfBnD} z6{BbuGOA}kZqag-ND44q={H?V+-KQzdj`IW+5}45;W~;px3^cEMPN#$jDaW5v1=_R z+7&DBTSH5Sq)?vx*I|axH~Z)D3;oNfVM0Q`@B7Dfhoj?;L2n;#xglZT)6*Tb0N>&qnnka+y4KGLjPlLWR@=v}Hvpv39wS)kue0ljsftM&93MZ? zO1-gsZAu3#uCkG4%5%|L%R4*R5Zd~iLd=b|U8qF2Z6t0ox|CMBv2z*d(?#`DiF;~ewF;XK6N_S##DHdfcGYai zRmsbs=M9ctPL>;W)8fHFZsr2ylS*-uOu*R-hDNH{1}%zh2)}4$Lj~I}swG0jC3b3k z9EAff<3^=_7ul!Tsa`yVO&g4df}KZC5d3=f+6O8>iK6LFgBBo++Mo2cB=eF=0h!F# zn0DNkjux|+3t#z@%S9tag+$b@u=s`J9L=eU-u!u}cw5@v8FpuZq6KIwG)h;T!7$Kh zxytZp7AqzMa0fnJ#y*_le5xZqev&x4OtA}9l@_@8M3RiuSMAwBz1<%ZJbFL088oUx zDTq+au22Y6U2elw*`%L@HLL~H4d0z|;Z139hkvSpDUN%=l3kJF=MY4(Gylqm{ z5$Z83b+*LSMh7*kKd&hyk{~cF$i?0`ZPQZlPU3Kh*vujD3R9X?iU?6X zsYJ7IoI!0!{udfNsz%e;%n5B0(V;0K`G`2GMjI~`1_Eu=Oi($_!e7Qp(8KCVXhwiF zhAZ_lUH{s9eF1ex^S)bag8ADXg{)%;?O#8*5WJda;ChTGev#r-Zl~P-JmANLxc+o^ z4P5=G*y|H+Q4&U%Pz>nv^SL`XxzCx5)%3m73vzahx-@~^MrNoezUU&+@yJ*&uf6TAGs-S#rFV566ZoRBY9qd4Emc9zt(m$E(dNT8e$0}3nv_hEo2)5*G_?(CY|kh;-*KLcF|yv=}}e$)?u<&H>kswL5`~@b<49 zxeiCdPfpbu@-xBCAnA3^DfN0os*IF{Ad|gz!NV=>`aQzBc)v{Z@Swb%3g?kSGWan@ z1qH)6oQP@yUR+YAXx|c(*f~4`+E;?9?k^ZIx?eanlu?n0c;+EI3rcl)uP=|bS)(O4 zA#W{glJs!9C*A3}sbyc>;3bx_)+N5GKbk_eP>Ad)xOnfD`YKXBv7=Qz;S z5$~8J&0R821v3;%$+h=t^A}rpaXVLde178D_=ijOk@yrxhhoEcOzd>@d(x3?^FsB2 z7#-7XNNy@|6qKHzqc9g$7m|>k>G}(4@=q%s+D)~my=R4-35+{Q%Vo)jMp@U0BqZ4_ z7^uOQ74n)5&E!~R%yk`l%shqLYu46qFnv|P<_c8H^kkH5kX*cJCaif3L&%%SL*9>1 zI^Hl&n@NoOpVnmoGo$eEiYO$%j^t-3EBl{k+PsEKzN$l>lp|U`R_A8}z4>^)F3!fG zn?LwD1{uc;34gl_eH_2MoSjwNWoRaFsf{!pS)c@6h-xL}%)Jjs%X#hgvwN&pS`e&O zA|aj4+%heHd{XEXZKu@DS@A~!0-X*;$Y}&!>`RXK`U|3^A?KtBZh!dvM&E~c1f=tQ z1}Q9Q5*+h0v(@_aQhpMhwzo81oLu?ws;>p__tyli)*RpbJk^e%0YYMcX{yiLOUf5sA;;_3Q#aa4U0Nhj9&Lx5xP^dbU-oZ<|`@oDd8JjWwAACVQY z+``FMg!=lprB%iJ%RvrkuwvIoT)bNP(p&IS*Hgg|4Sn~4KBwTB-tuPE-aalDf5b#~ zb#|c}KS)4cido8|X1SHi%E%FV72Pk;ewi$8J6wf9U&x9J16D(k6%T%i=${|eB8ZZc zU23a3^Y=+>RPFj*ny5#4<^X9l{W_^?tQxz}TRpU3CpAI=rxB_UT+Bwov5d?z0Gd^T zwZ(Ergh^$TmU2W75UIxaL!nUIMu1(qGsus2l$2`ttE!uIfR3yZzFNmXV=>#DzbmEE zD8_I}lSLKn@{`)Is%DT0CV9u1&4xU_9vHx{(MK zy289m=xI~m?9~&v*&^zUP!gqgO^wA*h$AtvAy*%GSqNNd(tmYEZtP0y%Oa8paNt^G>QTAQx+))vvlw* zxex8`@zMA_91$#b6P(4wI87yn>-O)q?@Ppu<*)mZ(!lKWR|B`U^!#PjVr#9Wm?prM zbjVKhYM-SI-+UQfEGEm?;QmC^0PAAucATgrQ-X6V&hBkuzWyKxJOJ{|bJ!Zx(Zi7+ zPmzaMUp+)*j=xE`!>5c@k;wZA{V6t<`dM>Xa8&NFbT()?RgLuK6D!kfa-E(3g&dvO z=ydw_tLb*yXc(UK$}$3$aj?%}ReIr(EG5(ClQ*A1sOY85P7$M9D< zDDmiCOfjaoOWu9x(I zHVn8MwKSBQW+$YWnmc}teK2eY^aB5*%v#}dvqlH&m{5!?e@We|(GX*Ij9l{p0u5Mu zS7ce2&HK#05Xh=g1(Ccb^O;LX75e%6sy^Y{XINk8yzm|>h9z7eLa)$30!4sqry+%a zU~;-?{P$D?N@;G{NCe~66M+gs&5RZBxHLoKBGz%!5RPC1|5tDJ=25f;FT0kSO`M@B zir*)HwuRLzE_k@axfJUWA^p*r4R+Lw=LQ9abWZ$Eu0+2ZS2*gTcarqd{=@=vxB5V2 zP&HT@ZW2H|86L56if|Itl4-9x?nct9GX335wJQQLH^u1Ih#qh7obglZa|}h z5wlZ1)J(H{I2;p$in~K7j)i~hobsR%i1yDdNDOv{7D6^6aSY|kWnm01@Bx8JcENXK z&z^)^duLXF@2#?$^{rBZc=@aZ&tI+Nm(8Pc8|Xi*TZ8|P3}<89@$z2N_~+^aP=!}Rh~c8yPz^oO$3C} z8mfS%;{^%mjwhbhuJ`PxC*O4Bfz#cNwP5~Xq!D}JQ*ZCU^ReVzDb+90%sZIuTk^7g zsR#dUcH&`A(en3nZ9xWFv7BpLs#lUF+!X&s0^*(d#;$(*m|tiqe&ugoK)=#9C$QC6 zps84r`6bzJZFsT_{)e3=(5H_AKp%@fzFx(UvYYAix;d3J826RZxY5*xgq>2UA8j$N zl(WS-dUX!2P73{J;qUZCAyHwWoOB?AZI94>o|^pZ%hlpueZ)vs`1qmqSw};C{*o1q zMM9WL`w8F*j1$G$q8B9ukNkw=T1O|yM#l=9#a0z#9Zph&;s!ncB19QX_=>bhB&JY* zq!5XxkugG6ypzhlI|3iA+9`j(8V3LBOem<5Upx8eZ`zX~q4ZA~G-HwF=1h5ve%@s% zi!aCYyY+K;S@E!RJcj9P3Oso8*mF7oWME=J)*twKJnd!l*};Bej1Pfr+MG~MI?nTB z_f{gi1SvXP^54+rt2E$%{S-gkK(j0Tgn3z*caoB$Os{X#?He3&m(drZlGkIfflzIMJ$N7-?_!{%cRz1&6=wsQv9u(uT z$uUto&qUI8bI5dW{5nh-d{n$zK`=Om)(Ga-bld}R~rUZSMImH?DH zv8~La{_&>O%|@$qnIFXpiJ z!#8pg%y^wNf1u;_6AO9G!?XCCiFm-Dem}>kB-XszRrE2w*pgP5Ysy5y`n?^zty-QM z8uDLCb|K_4Ts|2ChJ;mvG#Ku(N#fuozIz(K8cWd2$38LjO}S{iMKCr*;tltZ@NEYu zgD0MJ(XnqbXs56p;`mXT2_S_8{k@%$M6$YqaxF1Zzyc$gg^e7Ul{S$bV`kPty=L|3 zX*4A&Xhst&E@yf!FqWz>hQEE0DyrtlEV7cc^GZwwBw)*0YTMD`Xrh+R!ryZ%JYa$V z!P84l4popEW+E7^WF{g?3F^&a^t~c*IK|@v#n}b`4X>E~eek+1fmKb}Sp3!PzjGHM ztKZUP3{m|4ZqFTxzi-C}Q5GP}5$PcQc{Az|^!Rg})-WeWYDrCCYZFqxGtxK38&z8) zSv@N$`&6{A*$&(0UC)`16I!s&-(RYnHumk^(=F^l<`w-e;X1o19PvBP&^Zu=FnJPK zT=E|H`lxj53P8{|2o?{=3&u_2ih*U^TJztO?Lq!55+E*=zU$0|L2g9DM)eN+k$4i+ z=Jkofw2a`(lihYx<5TD~bNI$Ix6^dyj zj$qT6UY@SK7;65xi;?D)F%drf#X(Es;cnMZpXb80!)24)j|gTak9jR3uB#+tW5u37`S_o4eEUS z0wg$}o9ex-;Qp`Vw)=lCKG6TV`o6PRnYLjbLy5l z{kn62vvlolbh)ZJr{SVqWvg;G?NMyhO*X#x*Tk{?rMTxUg<3`MjjE@na6Q7)_lX|d z#i;r(`-h*&P(~{|qRBuIp(H5W5MBgvpTW>)ltgP_5owq{yYod%^q>9% zqd@R-wUSqQ5^Z;wl-Y_`Z3s!!$O%YQm%vBwW~^PR)nF5N&90N^8FJ?X-pUGj4p*ri@Z=GnX%`m*#`qmHvq68xLNIraC5*<{0n zS)FrIN_XYQ@Vz7$EjCoYZ*;NRN;=MwH$<3+t z);fE&`L$>0cjfNZ9^qUhy*fYIa7sa(6IKH6cOzLNNMN!$^yDjp|6AuvsdumNnFhc& z!Q(oeBp$n$w$%#WvF7WiZ9o?*8)2~#L%Rq|b*=ERa}_8O-p6Y3C+qt*5WP9oyLR=v zxRaz#PnYNBT zp?qH>Wr77WB(Nz?CJKmL602x)Ms$izHLo?>Fhm)FWQy=Yn};x<_|7BmU5?s!_p3_i zU%sHIQ!1S4)6lI8NN(~(vUoO5|NRfm>FM^fhyR1 z>FK4`xg(sVJU;9z83eg3TK#0cfrS^pwvTkvBE^AM$3Cavw`uNKI~nT2p1chCC-V51QG}m>(%1rAx|y}Dyk-PK2smDZIIHQL_-+53K!R8tY&=g zCNh)9roIEEK$dt`Qv|II4R+Td&XzX?6#6ddF9TbeP_`OQHAaP4ihnTm#i5Kjc54yV zLP$P&RWa4lvwP8Cvx1W$;$_6#4>-oMu8P=L)8QundX%v_vl~9loc0>sDR&*?(@@6I z)AL+3h@?>sVwlw`1^2n0VyfE8TBnI{sXp_5KjPV!b=qlU(wti=-ZfmHaoHURXd}!K~fx zJ=*nFUR!|{&m)8v-Fh4W#R>;;1-It$Sa$Dds6J6LB|N9Q9Jsx+4joaSL$lWPp>aDj z8Qcz8>&RSS!#|HsLn9^^jiffSPL~9@ufr30_4OCk@hpx>5uLq*-j}<_&06aA-mK1C z{(da5)MG5c5;D6s++ctoreFUxjl7;9YR#hu#VaxQfAlm(yL%;kzBI4sp^(#fiAh zB>^QdB%AjI_W}_&TC0^B)h~C%cNm$++uJM$w&C^PInU~P+K6pl`zTCMn?3?&N1B-y ztO*=dh(l}Xi=AT9zRPjJbvfUNowh{OTO+655;hTlau&W)*1||AN$a}>sWSI*%?I() z3IGe}uKplRl@|kQjK%DFPKw1isGxiw*r?CZva0QeFt)acV3c^sSGKKF3 zi5P1o7d#k>CFI0-HsnM=JAL>i*CWH0G6ZbuKH!g?Tq#@vat>Qp3^NpbTPDW%K1QhopHXBMM7}4&0o;_ymlTr zC*X%Q$7=!E>PzQi=7bua^7clYH)FA3`)vG}aLtljtt;!LcxC{wvq&(t!` z*@k}R!L)RL<~SuDaUtQ?H@qaTxB!dFEq?ZB?t5&iNEQy*d82!J@)BN%?GeW$Byjf> zJHyNyKw(k-Ke3v<>C+G+Ti;iOOIm0>c4fPX zZFb8VxN8~sTI?nte{d?z9zI`U`16X+G4{rmzs~?n%Ehbv8;3__4yb=Jiz?^ppMKpR zS=IFZeYku$xh%Y1%p*;0G8{fpUBs>3^uVYqlh82cjs+yiA zB42pUre1zBK#}5t<%C)uDVC{YCj&ce&*D%1sC*N@Xh*Eh-NEUl$6On-QQd;C=XIJ7 zSoa7V6^}1WL*=RMJQJ$^rz9g6M-$9V-iuX&J&cmOj8FLU4-vv!qXP-gEL)!|F1YV- zx7_rqo;wjOf>ySaE~D|X26~wF*^07y+)lU{{a@ak+B1BCNm3N4QrX^t_Lbpmb`pWz1z}KE!#HE0)PIZ}zvY3hS_+H$F#OdHFQrzsb__QQJ z$~C(L%xcH-NBts69kDfDnJH#c`8=;#W5G(r6E|CB z2a%0j2cfSf|8|V@zLZ#`y^bG^I;SA;2KJVuafiD& z1{z4-=MW|ykTE02-iTDFS`n5$LwsX2i<;ignb$1mt=|YvpG|D{E0MV< zyK;kfwSH7Tg4nYv4J!L>7P=l~77ZX*crAt>nzjgI8F4isw8530zy^qD(dII|OOO6S zC2{2JjKOUCog{}?h@-gaZ z=HZ>&b{Uf|Ty?2l&q>8LlRkv0QbbWQVp_ljWN&-idpCbFwa{d^PS%D`_Nu42PbaRV zG7^kep{w(tP(;;8co`h*@sBLI`NhW?H!%g`*ZG#K^D)6H)HG=Ss!7`y_eN8|L)%Kb zGWRvvQ(LJwVU`lclvu&*kp9`K6_mr({INHK(*Pnbz@WVg(eq}w_Mf1~-K-e~bNhpL z-AGp5L_{aT-QzYq?LvY1%vI3eyW-?F}g_pcH>@K}C~Q z!V`YW0PDredaT;WDkLQhIkj7jVOY>ZPWHi%wJM~l9&dPcxVk~jgsR#5OcmSNRC zztx6QjE#q-$xYA2N^Vy)3oC`R^vfYNffFNWh4>4}>+yWMmX z$cKaQS9tzK=dNA@Rp)-Sd`A~1A{+?mnvav!gcn%CitkaIXvU{l@gVr>u@vl}OpGsk zT0U;i(A|arP(ZK0ZG*R>7?Ugbjx^H@Q26Wc0_;y@UJpmrV?WI|mS4_*>Ul_;Xedv+ z3xC?^RywXn;@}xekM=rua-Bays#{Wyh{|tJ4!+pErXIo$ergZ!d4AoU;t zKtFM?RB73S&fjxqO{nXr;|L62bJm{IYp&HT5?A+ETQ#mmqxS$9(n{vnJut#7Y83fq zbrMv;`_Aio=AEP95Q93iE&^Nq9VwRnKG+ym`LCtlIEE>d1{hK>B>?O}J-X5PHf{WD zEO@l`@{krRV|WqVD+cnXuhs&BPp2<&9*(nz zAK7OsrIt-}n2!ta!c>mQwTW(hC~v6bqG<$Ao|r{;3Wus2vCG@>k0{3;yk?S{k6%@u zSyL$HSu0gwwe?m=DR5hpj3&>l$BJ%fUpEA>0xVkFM_SC5n#AdsH>utkS`+ z+m-(;R6niDgbTT84U1U^MKCM19YIB9C%!)eLmi^%Pk&+3<6|q!Zca?vGxz;kZcqT$ zMyXYJya$9s@N6rjT`Uvyo&kQAX_}2<|k;nNj5HIgMiE4}!kDzAi?OV;zYzX8h@Qd$`4?P_t&O7Y@M|*q92)C$Pmr`YG<{ zW}~Xse^s(c^g-f)IA_Gwhv*L*sQap=Na2nmq^ubK>g7R&&)m1;OXTm5H1Y8Lm?1e4 zp#neMPKAVg`6;|7Z|^HYKT5-VmM!58l83Z9QNXW-pIw>`a~Z2A7zMN-;zou6>EoS1BR&i=G?#h|VBz*XGfzBh2J zVo5AE+4pFbs<*U*M|mL3>Vr7Yf6VO&S~s4r5_?b4yCKBrNBw2G4sJ!|s4QSpmHUd8*~&)wx$#5q+uiVE02<|83!Yboe5Rb2U)mS8_%1 zGsgn@_R_GqB)cVy@iH^su$tu^!3l464dB&0QLtziOpp#T%zoO$!ltu;#`JXkHY`Qoc+!3zq@r9M)TT!Hy6f+P&0hmtW(2s2W)Kw_L~6VG^s zJ!CB%qmk&=X*F6)14?6AhP9M#z81No3G1pf6loHlz(u0(0lSv*=09<6FNvvrG5Py_dEqQ;)Gz-FKDV+yh?{UnM*-dV8l1 z>>oIyphH**-|$K2UbOqylhPbL(YsUyW;fJP*6;WDXDqFcq!@Bop(O7i|J1;Wp>7w1 z2ab-?jPQJr@n$5grsDEB_9|6$_Q5y>&#=FtnY#&!nAaz+C3=|357L5wAbKk7b9$fD z8p6$~rg!{mte@`jDqlc>Bqu83ux_gOPB^XGp!T8@NRPpt05@sAzn7V*9aWf3SzzL6 zsyf_ecL5d`MzNMkt1UGO-(cfAuxi-rF~mqj}q0VDUzXtoKCy<4^lW~ zf-Q*5#L|=feZCXuLNHCYaW1%35R&OSRZlTSSTSgn(%S9K;DD;iC1=kSC4R~;jAf;B zCx{LpD~nU3z>MW=t-IhG)?p~kUxz6{>Aa039J&ERv1Mx^%+Bs0$t!x0l?xK{+^L2Q zr+Kv!bdf!cQMyKT&QuDsCxkZ_ES^AfA6Wxk%PL31vH8acjTVIv%$JCS7tl7RsyJhz z8m0o^>)T*Ztq(b$APe=8c)Skj>eJX>#$>cva(9!ZhTq)1U2p@ww!s_obXDQ}1oXEa z*ZV7Vc-B}Z+W@MLg)q$E*JmANb4{19JnCDi^v(n)OPoQ1&OdyHmp@CKBB~u#ePl(D z3v~Q%iZK@FL9akQq!fRzpxpaGH9K4R$r51y%*w$QGa?xdgU0jvI>tIlEP;X@{YO`^ zt4O>Mt4eQiRz@cu2tA;wu`FC07_>v{GSEp-^67J^bGW)**X*}h zTjt0O(`!0T4M8lk!Y;R)wr_FU5lyX>f#q&Xi`C+IN^@<~|GH_zpZ-kMy)*lFUnE>+ zua`_dBcNcslE>bB<5ry~mtx1x>+}b{ft+n9N#~M&J1=FlT2wbLNqRff)Z?AJPD zQ0$5QX)+5^k`rVpU4;y*k1P^#)x9`7X-?WgX9%p*_`3yWy}3TOya-vwtR(xdI%BJ{ zHh&%xv}d7NV|`CsgkwVsF)t^5sp3*|cXP6hDFihU8kLiFXOZ)&Zp29l$C`14W=z#5 z=IWrDSpBt)@ui_+{dP6c)Th%xwhXcZJ$>Ue66x3-0yXE?Jt&@%*vhcYi%3GuQAz~e zipJ53;a7*4{=${kw0;2Wmb;vx?BlcX`<=*~n~~X8XdB%u11a=Ujj-H>S`7KQfpTb7oNO@ypNJwrFyx< z3v0BlznjJBEbuxkaSPcY*N`A0iw-Iq{<}TaNJ4<}9venK^Guc_dtX8=%_4^x-aBsF zR4$4)6Ykld!6qc<9pz~%5{PQ(JZcS|7E+Q1v&_vX2=(Ih6YWa zT~mE-_xyN1F?Pu-YcXFpNIPbMPmV2ja{M#$|QvySC#5flT-H= zO#l&UCATD@K!9-2=TuvGXjrZ6z&%A3Og=)BzwjwuQ)1eq5)&4_L+8Y0QNA(` zK1Pm4HwDsz+?Q4bYgFX88jNHefxP+(K+~KDt&PwS$PF~PFrv57hmasEs7jMsPTn!5 zaWJh6ck-QA{+T;(jV{N*c#}Rdy6MfDiJwB?ax_yH-}2OaqoHSA*bMB4y^^Y==w#NM zs(H_G=Ta1s0gV>T5j?xsp@~96J!&8+veMyhiT%Qco+u>$dYO1Cf1yfz(ekd_JwpZ5 z8`qgJNH^t{d2))kPs84TX}k8Jiagrf9Dg~v$te`k(5INn{D{%bYm;gkwFac<>z%Lo zJwnw8Bs5FdEcu30=eKtY4sm!LSW(HF{Q2NyDBWx&>@bAx^Ncx9ky#iZZ{f##H_wKt zD>s?C9k^twis;AUsu0dBf&ti$6FNey!8EHy+N$AgMW&%QNHAObcdfTe-h1pL?Ke)j9>;+*D{xT#U88--D*hE7DdA>5uVM zo*vZZ2a1)nJq#x%0n@V7V zCrYR@uBx6G>ekm0G02Oc6mT9?>))R+V^u@7%D<|JMs&y6wcJ}>xs&XxN`S2aRaQ6V zKGS!tdW5B_SM>2F0<0x}qMS@`7(lXZ&pW{KHE?njmrX{^>2Uwf&5;krb8`#g_b)|yG zjw=!zX<=OtEN|lq!_6fc_?}*zVtR=n4|46a(!k9oYvt#1E{)a-xe9gYT*Tl)usSd}K1%|7y(?t;l zp?;fC#qzIj)Ljf)2gjEEt!%LfwRM3?luWeYvFGu#WRr zb0xj=p&!XdQcW`z1Euiz4ulbK`W()Dtfbu#!Ufm(=l?%igHHBEi=1QJK3~nhm zd2GM_G1A-MAcJV>u_zH!87PVY zrt-`}H~l|c`-%wTiJ8AZE8+D-^6*M)u)DzIOo`-Qi|cUBVp_r_lUwO+E4X(`Gwz24 zno6VyGhq4~Qum2%eq#NN84JsTUBtlE(Ky8q1ROv6qaY~YQSAaOQuLyyMlor@-o9jI zo3Fy^i)1!8%rx$wz_}Ir_I)cCv=?t2ZMmUw25J|1{~t;aq$Qq$6$BNKdnAHLT-=4* z*)oTv0lveIjUzNnWD1i_Z10ioDx|E|nsXhhHEF8~V?skJPeJ$Pp1Lz~{b=|Kz2})# zle&SYyoBA6$Q6y%X<)6M$x0jZmF00K^DzT{9qzPe#nLPnorQtXZeE#Vb^KbzpJrVCxP>27@JT(4q6sUSxX$fUmJ7+irMg%W8F$5*Igv$i$(}GwRA6Ax(|Y~nH+01hELcm6`31Tw`~#J8)MA|MNNrnM7ADQ zYcZ5>ET%@%OcXp@so`-VH7cD}w4EY)rRB(6Fw3Qb=3acDsfL?zFeYXf={e;XYp=!z zRVS(rP3;q07iHy+73R>7!MypZxm8KL{G1fT$=HcZUMA{Tl9;$LpJy%D8Wu?8?7Ts? zS+y+OgBB|b#=oi*$MmVuJ0$6icx?=*QpV<*H5DSBpx8OdCW+>@Duu?q#K0Bgj71!wxD7PZ!oT&=53m(xE19Pb=c5>b~HqDr-z zMz}#|Fj2Up&U!0Mg~o-Kf8-XCnpsG|I}|mas|v8X>}Nn9Y<}chxcCRdsfcR0MoicF zRX89uL_t*rRMvG_=q?MXEq?*hvv&%E>Ym{5Fy}XW5xNR}-)AMm$&o*eJb?C3l<&c@ zbYoHc$1oB?wQVa$@g%JqMGC#-^=kOkXP3d6xUZa7T#$2U9K1Y2d>br}BAJiMoxb&x4WKaiQy(bO1gi{Hq+sMkaec^HH|4j#YKAw!RbqmA{Ye~E;oMOpVOtJ}# zA=1n2$jeU3{)O-K5KfJahzy!MJBMYMm}dzgojlLVp@W)TUgQ|UGDoS9!RcrNYqB5X zAo}xK@QG;whY*>5LryGWD0|aOs<#{5dTF@x|MLMJ*euzMA-&(f*FDkB$E_{o-~S3+ z+V~{nI?s^Ev!~O0C{4|-D|irmi8pq;{@y{#7Ih&d0tp8sZC(NP`UW|Hpxlx{DO#=_ zg?6iO;@N>@YYD%GBUE11|EM){owRJIdRi@IIIkVtUnMlpgoFDZY|A2~jj67W*KVOb39CnYnT;|E0$R zQfE=6*_I4Nb5yS!3kvk^l ztek{yW6VFSyyKtIRGt^k9SMnHxz^XpdtUq_ONRgGxC&&ZAq~xC?~g<8{qP=BP|gZrMUtzrY7j&jOuQo&40j*FETlar}hyW8_4`9 z$GtRZbH#gircuTe4A)5Cp==uSB9z`)&RcmJZJTa+S>TiNKrSTi3Jzk88kCm+rahTN zSeIwMyI%fWr9mXhr3cy7f0yHSoeI{#FEaU$VLoGqS?9{=4Qr_T&^;S(rFY_j?!1ns_gj0!u`r#H}iUN+8LrgL7J-QM8V^?b}h7w9tOy}t+n0%$JRc6pq zltu3`Dohy;N=PY1BS=6pI;*HlZn^B{%Tl?{CvvX!xH)MqJ;}@vZ-nQ8Lz9Hsk~z;V z_IvrcTR1dxs}3Mqi3q##s?wLq*y_tGxnR~L{Zc_Y*M%p#+bkDTw#J>;0 zI}72R9Cy#hvPLUO9j1y+Ksu?JTylp7DbkoJt~#-~0RUCLgnGxS`t#ztZ>~BaH$ASY z*WL(+v~X%BA+rx5LvGRi(oTRn)TZ2cl8;%Yt<*AVg?b&=1+9O^#LHt-Hs{{{w8dK+B6m_t0MmLX)L|;fBcj9akl>&yOaO-;gg^@foV}GeemcD6$$`aa*RE&4 zOHFa5a0iOtPGZuzM!5@%D|m}2S@j1(kI*Po1L`@&5R)pSrZ&03fw|J%xPLOW39bF zQs=8k3}G*=IRVTf&u>qi;jFD+O@*nC&oO)a2(wlaIhv`WJP3f#p%3Z!aHRtc3Ji9p zV61}oq>h-k0R`jX?pEC?NCjJ%8ECKAjBz;*XuC(cg9(h7KcGe!kugf3Hw0p`48{HsWsgoXNmmG~{;nn6hXd4gYb!>1I&HN8X5b#|MezFK}vJ=XQv3f^@;q7H+Y zZ|oG^VoZ||`A5g^#0GOtF?zh4;Fu)iLqL4*9=+Ic)B~^BUOKT)+z}wQe%Da*(64t? zV&Z_&C+6*qKJJNj7bV+So^NFAp8R$vonsM~X)wH<{`a7m5Jda~))){<>=WAHH;9RU z;0Fj^P{t!Z6xDnf)w~zeyc7EZmLwoavB40;Y2J)QS-%Dao^G|OK&XJq-@5c zjUszzV2vVcs%dVa2A^hna8EaZH$GxLm+t~pN`}jBIj64aEKf4Ybr@^~Z{F(|!jJWkZY~_Y}H6PC$X_Urw)*$gaQj+7h_2W$Wdgu~u+& z8PFrp7|V;~0)s>fOjAv9mie_OtTU2<|H0x%aK%@kMIFD?A?Zdb9l|k6EUdo2UZw?9 z6lKhZv@&I)>St0i;_{lXKN_)p#{r#`(xCK?vzf>?VYBBYT_y0v&&*ALEm`yqQ}ZMn zfq|5*V|Gz%7@d=06&Fzr0$AQw#MDZ!&brwW*j|i0-HRR4neD^B9%6}_y`5~eWWl0a*GvC<cur?G@1Mo|g+YiNJ3+t(XVFkKXqV;&R;sJS6lKJGNe;$`DA17*_440BX12Z0hSbo7(M-M*r2el$QcU5?A+Th#tjp zm$W^$6}rLM~mK-@MG2Tju*;0pQ!14Bc=amn6Is~gqps)H$+M28zsHxS=Rnc zN#}c>dAL~K?(!w`KfPYR+{!wihzol*?IWn^z5gwK&N7dbBJlL_KA1&G-+BFaEJ`~6 zf9C!h-g%hst5Ihj@;5kVKA51OQ9cc%9V@*mEA_XvyL%;HB69lOwi-dZU!^T2UZu%j<}Tycw7( z9^)H`a%%FTVg84`B^CYQ71twGek#N?h&cz^<{ z&$P%80ZeR`QvlP49t2vb+3*)N*Z}AsThJ+wJ38!l^{R&Lnj3>2!65IZCtyIu_?Q{RELeH~5~XuHLzOfAB2+VDbqiSC zwtJ3wcCX_K)AH~^`+&@Kj`-U}N8sIatj^heqeBJ`KfEeUQFQV0MXdH@+T{UT1+gkS zJ8@SbFLMTQW#Xo}vi^o>lLY0=E5?C4PUsN618Ho`O#H>k=9m?D{VwON{1r)ts!_4v zEYX{95So1yoPF?*xjyLno(%v{i7?+G@X*)&{C9{ zUt^KLT6bmAI9c$&hr~(7yBv97_~*LpA)E$IVLYo0j4kP*}ODk49cUmNa3ik`G79 znc*z4b*BkrfRTpJO;PBq{R~)|ti4sd1Tfe!i0PPRT4GXdRm|sP^o^midQ3Nl81Ai4 zM>-p#BkBAS%Day@cR+cGYsR4R)u^ua{7v0IxO|e6snSCD3wxn7&XKgQ)Qd84HW(-n zwr>*U{(cSyAwQw1U`)nemL+S=+8RS!>z5Spy+EvXr@X3H>K>O=!*(J=3-ct{xjEVs zueHU9TYDW4oH1q-Zc zFx7vpfr)G?22_%5mhXS9K@_R##LH`px^jMySh@-Io7$>~b|6V`Ae5}5$l86;xSbY3s%<)JsKgX~8y*5(5q;5aA zo9M2@tZvnm#+J0%%T0okS?&fqU78iV18wvyN@c%NKn=nfsfoxIV3IH&Bs4J`5Y&cp zb?Xm_p`67hDXnP`u6rMB9)*j0Qe8V3NT&|H>bGHMqpAe_P%xy{_3E|B74y{Z1j{{@ z`ByKF@ITlfZfc27RynQpl>C^fQ(S3UT71`8AnEB1!~EA-W@^}O5`aZ=#0)n*i_7{C zpT=!~r3pbksm?y_y?+}a)q=R4l|_P1nT`BRQBdQC-WRybWY~v}1tN7Mnf(tTGZ5#z z5Yc1iT9R2+_|&il*|n>?J{`)~)6C9?ni>%wRohQciHw9TsrV9h0ZS?qp+zY4bgp!{ z#e-SsgF@Vi^9kAWb_;epfoCIIiYoh*{k2lrNzww#xioc!UNL-{vg?e}cd+Wc6*~dN zM_qJ!;n5k7AHl0ZH^)0ttRA-m`3R&q!WOL{Kfml=w9S*C6hJk$9I3~-S3J^X3J%5y z8Elie{w711mB$w@MLajqpseb*9bkll$z^9sDz~y$S_5uhfCqea)ue8gj|qi}N>n%n zM{ss6U=rq2O+n#rEjPZJX>oWwuH8wl)?&A7t2%w@e|i?Wd}T2bukp1;H-kct z-2@ePV1#N3$G1{s0wtX(5yWKBLmC!3@UUgt2OM5syDMo3Z}vRO<(ydN`rv=35R|-a z#Um^b4{B58fS?9#!zis#?$FX(iB7}JJl-^ore;3SZK79 z_OR8Y9!!#-M7%U+ysoVfuPfa=XLIExEzTD~1sNVXj*K-2g_Kl5rTGPRoNG8=rWzVl z+d8Xzyn#*9VS9t+*eW#Um;F4uv5C<3t=8a#+Vt%%D{^ zxE2NwuW?&IwK-%Kp5~okmND~MzZii1*QlwFk6QH_4r2w6tUQJ}Sf@KoUh>ckQVpG0 zQQ3SZ0(O+s`sP2{;2ma!ff;>&i4?7J+MUZ1)Nu9BL|z-F;`C}ZXuufrAJgkFFa!R! z#K9SiG7P)QK+uWvAX7FnZXXZ(1^;+=5}50ha^VFrl((=Uk)hJ5n;w%(oy^UA@ANnVdByTgA1-pkS^2 z00K#9lz!0mZJ4Sx@)8p)Gn!2l&seSK6U@ciJ~y&uDGbSCcedOuqZr_fPHN zFOht?gy-nJ@nv7p`VpB(rLT^7bxxxCX=a#NOV>r1VdUi8_nr3ia$EVqt;SC}nBQ_1 zS>T&AoWdJ!{`*A~7Njz#41kEN||A0Be^A~`rLvnEj6d_F=9mr^+L=c}{+ z?|GTfhdf$(RwYYK(c8gbA87;2=l?GyGWu!ARro|DthhW-{kS^Dxx_g-6dRF~VQo%aOIsaBEY>=TN zq=z~AnI*r>f#vFHCTMA$osSkw;C9uiB(zc$o&=e*?98N;Z9|;GHlzSwctYAqm6Njo zFX1vOZspY6{TMk_x6j?Is%sflH@Cg)s%wuq6;p`3%1OC3X4Q?0y8{(d^Qy+CCmB_@ zGAeH2x&Nfb`IVEiDyHSXxs{W%hzol(9?vSK=5Lw*y(O=5a<>0z3&p;Sid*-;1I(%$ z|2_Qgt8wn&M_Ed{(T*4(Xhy-m*)i&k6=_p}yhfNr&7CMU^92F6UkMp}z${5WE#U#L ze;e6IG3t+{%78S46V~qfs@$dePVX3Gz|Y(%!jL|n^`5)Hy~#SL8HZ+FlO$PTHv&)+ zYbp_^DfM-P+c$z31x*HOgkzCMMyQNKtYnj2`K#am{yBCG%9c?4!YGv%bIv(WI!Q4` z2_Y-Rg>;;$mU$|&%b@@lyz~%XMS)19kkz<=`eAA|ar02JIf}{yT6WbthZ7ssq?lH+ zOft!VTR&?Ek0^z>SvwyajMLa4j7BAS5Ta&+!SL0S953w5iHaOVn9$I`G z%iR|Z?HN!9nDIOhlL`FevjK=x7gv800jI}?+$9{(S%*6)2zl-1I4~ND72dB1PeKD#(ggmRD);{(*0hCU-o$DrZw8U@fL?w|D$%~1x?*^p-!g0YH=-`)J2Kn9D7B^Iv%%6rH}&`jj#3Gq~M35J34tmtHL&M+05g2mQo4Gy~7UVGnJ zV`&K;E2A1yrDu%(w@Ju*_*{W}{+el0FLB)mz?93gYYgBsMIZ{t! zYYZOWj~wXDGQ+o~fa!u=LEFT7ekt>pu0s0Xb7%B{fY8yvrLh440Z9hNak3}YRms<@ z0TZ^l2wPB6C(bSiDP&vk+PQw_qSn-vVis)HwL*ny(2X~JG?n*(TZiu2G_1O_T8}t4 zrTcNx>>>GjN8_voU!K5Uqx_hp2&|@DLQj7oMTm%1Jv+(AcOfd+V?>7<~Cc5Z{OUI0(eu&j`!3LW2&9)3xFKTy+rvowuaYs4Emt6@xOfUVT#oOdlD>=VPE z_!?SOKP{jla-i9?qE)T0?E4=Gxdy9!?iU3N!id%QjOZX4(1;oS9pp2 zq>fTxD}O+{+gaHOeMVTZ<1!K7c>bw#&zwe3&2>EF7O!oL`RM6EWz3og^tge1ugIV#DBe62j}1ZU|GGk!koytm#@w8KRo`tw5`ZQAp z-G}qeA$8VXNnT>cx5e|lN&Ybg6*vSd15b$p@7cZLI_1((a>W)QGi@lyEu+jTBL&(q zu*=X#|GEBF=-6`~<0P3gb9zCO8@lJ%9Z$`r*T8;~9|mIkeH1&N$+jTi2x|<%#2)wh*pww3BO235_ z;^+GDk)GNan47oaH&e3daDoAQngm~d2^(uRyG&IFT}o$K?S6k9AbCD&ceOI1tCs-~9xC z+NapUJ@jWsAjFn_M3P$fPm0opLnCe=bZIsd2~?0sNeg-juOXj5 z)*}L4Bk~AXs8{JE<|umcNU{@XA8mfVJ7U8<*!+J=bGvi<`PKt4vaDKb8ez)zS}|&* zMV^R1!AWQ1d|8^+Lu59$`-~vP$!Rf=U9x0B68bO1U~CmOQWP8~qqC+XlDHO?S?e?P;ZtSZK%A zY$yvY#Rk&$hJxdZ<>v-lV-wW1uInr8VQn;5Pa?nnVE`VzlQ-UChYd@w@4bF|O*gPLFiQVdgjvNBtc* z+%5a!$^r1DuoQ#YZ%m zu@858;kSv|yP?^ZEmjnXt}&(dCc~R8y}FIsM259y3d}U=0M=^^x`^n$moSO$8<&Yk zq9%zeHp?OZCicsb+l<#qjC#_U*Y-~!YHv|w=o8YybgiTTr2*6gT}Ge5yAW>6^>wt_ ziBxMi?enIq+EOFet7xxs_bp%$H34Q}45*nRbp-vNZ*+OXJ|$1;+lByM5saFafaO-D zwuK(M2#SgnnfHDYfvY;1z=KJtOl@JL&IELv^>ap^LXw6N75vt9kVPfJr8aaZHj4~f zJ=>a2I6r9@~XYu!rcs&G1qBDVH^pWn|>VeQj6_RUX3=fs+ zm9Mr$WkGeCTv|4y+INtA!ao%>{C}Sp*zm9=TkZu=NV&GM=%zZ%THLlb9Gs^=)r{Y&530Ah+jr6hYyL(Po={Ci-EF3ozNKO4jF|&azQP`$A* zB>p8F5lXWmKy5eaJiZP?(xPjpv+SrWY&sxY2NGw-JvZpgSjo#pR5Vwg+~U8}R7eY2*IXh{b+?S(okH03_OQZgKkdiea8FjrGtO@Q z+_#My$g9taq)US3OT!$V?*>bsJ?~WWxf(E7wXLiy1>PF8UFpNp-x6w;w`trpaDIoS z=WFx?a9m85-kG+F05jXaAluRf^+rZU{`Qy!2gcQ2FdT+Wr3ZL2`d0u9UQ~ur8*$<1 z;-iM0n$ULuURl!Mf52_fn8Soktv@6SSGysB(G(Cfju&VAbgkegj()gN$d7ZEPyO}w z9MK~Dn5{pgm|otIjPkX zwwO{T@Is-utC}Yf$4x109NGG<%wC!ipj$u=qJ{v$FokniqJLHUU$eyg@{0Qb{;=L* zvd{jKf}T+BG-c;+%~$@0zS?|_!s*mTY6oFvYCgq5^{{lwd=0)G&f0ki+6*q(tJ-6y z_&QB91Cm!E&k8K``H{g~w!OX-SKiP@>j?X8!5_0=TjtAny>7%oB;I~Dfl4RM85@8S z(_Uv}oJv|H4es~D7u^9VJ29~)dM%->*3dkrX4gNE_#Crb;Om6*aFO;cStnC=x#$J8 z@u4=oqB(EWK*bGOtE>C}I%}lf2X19_bKCQ!Ix0eUsMo1mksQ`~K{fk+dd3ng!&*@2 zR+UidU&C6~4%n>qZVI%V{qKhu3#%5Rhc2v76cS<5QgWk5_u)AMfjnUy1VEaoN)&ts znCsYQHNZxOy9dR0+zln|Coak18`^q~z}*kr?&p1kpKN+RE3NtdWp?HGq;FwVBAE~r z|H5dLl-(-R*=HG8F6h2cm;WY|GLx-dq({){ZAJRz-b{xDkTA`CCBV`s(AuSkIqG}h zi0iKO-y7RHCU-Jc;{0cM&0L0Ks>Ct=V*j5)gS;zdhAkBR*mksy-CG zk8oMfLICg06d2BxGsm`)v9~W`0?j-{H6Am1aW8lef^W=PYTn%|-*w$m34+XLe+k0- zTnWN6UoxKnnXiG(FPX2wBE)Ms!h70NLmmVT(aM#JNBi>Xg-Zm|#wZsaxDG7fCqhv! zyfwNo_`D|v-WEeReaN6@$#809UeKv#sc>v$UeGgUsc>jyerJtQE*uX-IK7N>^#bwC z==}2g*ya5EvcyD2YV?+w602vf#3Vs-0&}ZRav~!o#+M39&BxnT*=?6Yq&{Kt08yE@ zqebp(;A@sFDIo=_@~cS4=N&kEQGLzUd<#`j;G3l62Q;(T(tTedcTMYGdpb{8R_w`& zxH>)p-(MZPU@9hq5BoFU8!2CqF-XUFx&)LkraRvVf+-&M7Z##YLES=Y>Kha` za^8b=P)27h7S_?e73)*s#k}YM@-lx;c@gWGmeEjdIlkkr%>|x)3mlHi&TT%Ro<^Fw zCyn&i60xJxcUK-DA7P}?wc4@V3%DN@;##$U>S4c!uRnU?pUod!lckF1Y>-m|rxDC! zW>Qf4l6qK~NmnO7wh}~>mocB=FTpy0szxDx(^<%`O3E!$$FtJjuE`{)#3F%N=)Gz) zzMWq-7%_(l38d;-a>%{2(&!E(;dQ_0kF(+m&b&?4%l0Sl**2I~kkmNXz`7)@GK4dL zfmN2SzUvQbt+)nmgoHN`gs085Mo07!M}>h84s$Pi+z8ZcH||NcMz5lxu-EV$+PFc9 zj^*w;@#1YFV;pZ$lV^aS<={`h)9>*Ylv_f5=${d7sA1}USBU^KLYv<;rq zX`wUhR1}fjg_0puQim&wPWGv&*oHy7M^26Ttm!B} zfjK>&={>p7Rjkw_Fpquy9$a6{xd0cgFa4&E_`Wau{7f5mYu)%@`pyymj1o+rQZ=X} zm+iiewKkjEEiQVgkRD%rB{(bZr!=iCiyDB^L$A#OnChik+Qa4bLF$)@g2Mx7a)4mg zitl+NKLTFv1Q0J)0Y005GH_>GtvHi_dQ@tmZr6zYHo5l^d))r8{WbGLcI2K*>^b|K z{h$zGyE@$oYc#Ck`=K}VMlXGY9kdd0P(8;o>+4Mr@U&0g*dzsYnQC-~Omxr!GV}ro zR`H2U6E`pUnS+T9&rDhd$8S+OS9^kvPWO$axB(C86ttC83XQRfYbxbqxub!<_Y%jk znvFc^J1T%_X0<0``1<+%@O5|0`pkX5@iObpdk#6}$H2d@o5%kztMFbI2>net;@7iJ zd)oDUrY>_Q^Hp&au$n)kE0QrfE;Ckw{M2;RgYhES_%_|%WuR>)fs=YB|$oY|7 z?3~thf`QfZ34uZZtl+SR1pp~jtIyM{aXFHTmHIhdE4uF+nfCD;`+IINRYL{XM_=6w8(zZu|rF< zcu4S9t1%)Eu{KY==|e#D&ZW;*(1<}tLNCX?^`p?VRYL?8V(QR>n+VlG)GQl6Rqho* z!g5$FG`{@NIn&Izf~5i4EI2&H=&qXsNo)KXas7GY9d*53-}q)o1cbVad1!U@)Bq=I z*~bC%jbH688f{UYt3hCM@~9kNujoB*53_PeWi8mG))JN%N}$A_0Db$`Na(m7gs|A7wtsps$zy z5I!ed{cR$gllSuLI-JvNIZc-GvY^x6SNMbgD=T)Kt1|){y4W4eXs_sC*X%iFXBA8< z<-v}%o}^~k!@h%&JG}0L-)aCFWarz3aRtbjerpR!JGA=?e~P+Sog*fU${6{jxQeMh_74; zP)sMTTF_=jYU!0Ag9!kjzn8~lemsH&xwCoA=pD{$1JuZi@`8iP-_B_qZo+Y0PCmr= zp$-=ph{J(+$DNIuH6<;S7nk@99Lg8D7Cs>C$ovtql;3U$d_p{}MH%8V4YD$&q81GR z32nqxv%C8Y&!=Y?txf(c(xYS$XmSmz@MIMDJH2z4GlmN;(4VCOp65Jjk3@Gpb?VPS zrw&@C5H-LqY=~O7gRR#y;~3L!ql0`BIG99U%#)%ZEH3TTt7V;3U7^7ot`JnniRju4 z2~L7HO+K4h?RDB@>)Y>cIHr}TZQHTvXcyM+G+uY9h)6&@M5n*o&^>P9|D~|`?-NHP zE`^Y85Dz>e%5qx%cd%DB%)|}yOaCBoSAz-cul5klDbzbfMyUsybPxBDODX+7b zqaSKcLwWGi)f4QKCY1t67;6RqKKoSf?rXuAZRsn?-e>54Dm}HF;6_^Q zQAXB*@zCl&u7a#Or}R$fR(*Jt0yg#PQ=-KL^P*s%e{&YOGiiw%u>-nvpZ?m9Z`q7VgC@D*k{A|*A z<4ck3YOft{(#Rm)ffp?28h5+;0-M!@=8-_Y1h0PgH?+j03V3TMSKcpR)fXWWEI^on zg{D^FDo`6Gv zDfH@Ps@v`NzW-e6?ku;u;$kbey7Cg?_qeKtmB_D5ibs)d&tS8(=T^QEut`C~l{v}_ z0Z0EgH3Rf>8ABRpufaZsqQx=aqZe`V`FD6J{aX`KSdp9BT#x>3j6Ct6d0FZIoA5pM;@| zGSmoPluA16z=j4se6+k~{iljZ7UYtwUY>K1tUgJ~VB41>(AA+F)qBH_B5>K?&$ydf z-oUi9w{1L&?SMddcdzvdCu#X_?RMAPWT=spYGbsuo)DRI_vQt<`kq?j@v5&PM!VWy zrPxiyRh!2-%Q5N9{P}AArs?tOsqO4~Yaix8_hBpw``x2!@3k+vo!*^R0_s2Dx_aB< zcG>^fcXv1LAfK`aT*F@y#|a%3wL=ID=03h{;*6Ik{~?Wg_rZ;DTqav+|L$ z8Wz6lqSf4*WW=q)xr7!+Zx1PvVm1cgvQdWFS#5rv$cnS5zf0I1wSPYldAsuOX1_xt z4{q3C&<(;n)Gng-*AMOj)3seM>ietHN(_#b(zrGMIzyE@G^kx6T>3gU!04J0f_Hqh zQfXxV9jP@?tlb&k{MemKbIPGrJ9BJU=nF?Ws8TPb)B9~I%Cz0X9lh{x!-e|0O;7d= z7^U(a0T9t6UlDX);etWtL>4uSvdf& zCZgtr=YYQnjrr+#q|0=R>(N=XeIL#dQA#oTNY{6+rgiCS$g z$x1UWqZ{@$Hkx)+Z9M2)dlalBPH7qp-aZnRE1=&Jv z6*Y6yzF^2phXT~CqZecf9+gzN(#_W4FNdJ+(0Jh)zNO43_S|cm?9|05)VQ8K)IDNAatcjuHRQ<V74LBYHd0l}f*FJ2@9E6cBWPb=^B3;v3()RO7jFe_*i!fQ z%u+*zj;U~|%znJ{T2}jP$g}{1Lv0lxv6JM44^%A6#ibE*V+q!&&Q=hiNo(^qbI(Ff zyL>`jU%odF2iNWzI*za)g;oV!ufzU_%{rd9!FmsR}cvu^*!P<8!( zyUiauN0yWJ#qqLw)d{{dzU4q|fFhpwPPMoVf9o~Z7`$DsQ`dW-!)*DaBs_PI+f}i^ zcEGIll6lG;K-YkCxIUfyVfu8nzksCj*uaJAX=Wj;huDl+E?qlrmSq9>^ib1L>@B5! za~S5R1Hb|O6SQ@62yX)~du&rj25BAOi$t_m&R2}tK_WWHd8erme%3c}Au!#?%%R=! zC3M*h6$BFeR82OP+&d;bjM3|IN=$jE`?7%aK&SBZvwFa(Lt&~)gPQ*YK_mOf^lPF~YCE!dL233k4jC=6%{*oC5})t)aw zI-Ep0VNDqzN*Ys{MhI*kDz)on<{gk;O)@aJw;Tn;3YT^!aE;{X*{?n2QcYCBgLGz@ zeDxXeeV?8yKs|WAIYa(Fj$f;X`P+hi?mj*)PeYvceZ74C-=u;+?~sbet8aDr@qPw% z{xM`uK3}T>f!*xeI%k=5=3GITV(I)IKDPN+xZUb^@vtirV-LhB4@C<**Gp|^niTfa zz_{njw%&A3G-pxM7o%qI!9EMu{_uDp?y2(L*Q;CBggHV@Et_dJO@`Z5&1gixQs)1#SnI$b<0{Zq>*>s z<3j&Jdib4LkTr}?7R+kL2^U7YI zRMWmtttb>HVVYWKW_k%i0+qTv+WUQXasf6h8dYAst)+fQ9)=(! zh#u6gJsKodz1>}e?YZ4NS3F@^43ZV>EP{~=jv4d@_C=`h1v!OKC|6j)0C76Ko#O-U zptYwKM*5dJO#Gu3z#-)lR(##Kj#y9er2HDN(^6A8oz33tG%$(HGjYE|sq|LvcP4O) zrHNl_5LZNtB!$}4G(7+b388fqqqBLHg$uWRzCkg2s{dp7gO(;+zsYcJyQa-NeB?;P z$XcoL_uBhga8}=+%lq?V`}iF9VRTe{jaxbaWGS(mwj^jPT?eMt^mK(Ycz}vR66la6 zOc5COsE@&v2?n-aogLZ|o&O>f2P;=a3n#R zW~9&#ZYIoRH4_I;^p3lgN#+BMs=GpEN(&14LB=gO`TfJw~_)$pT zd;}NBF5TavRTm))ab}@)l>t-{_{#=lrJxMxvN$LMHw)ZB&fx;U7{_7zX)aa=5fN&0 z*bd7%Y5S?tUy zGa<;JS9lrlM!3Qc@`dP1fB;y8sh~BtXaV^u%b|SkzKVTndson|0A}4c*kMW$X(Q={ z%Hg6$>2#GQ)qbb6CZ~}L(q~Pri0v24TMfuBs3(LOzcBBhJ9VpTBHRFTv zVAfV|OvA&u{6F3SXINZL1Sgu|9Wi-4W9unmx2nWk zCAzVd9cZ>5^wG8~_#xUM`z2PHQ97l0Q zAJ){l?_Q>I2pKBtuIDD0>ZB;sixCKod!Dgh6^8+EV;bT)zx&`Yd_zFvv)*^3OUC?!!QY9f2)@oqoDeRE`P24IZGsG&*4>LJ0Y%`XnA|=f;7cr=k+hMO`dYe&`D4MFbs4 zoo#IqQk%(Ru#ZYYu9E_JPn8CVrZQJHe;1gXHz-fi6lMtA*&%to&mHJ74t%CXDJ_J8Y9re%sxWHNXbog5YA9elJmVIILc zJ@*aIio+$QW1Hq7RY7T z&a&9KhDcWpKYL$iW@I7=$-v0MYHgnSi1!es&0?KBU4a&BH=XFGy%;{YuV5DkC;W>q zG|)EB8hrYsQ)VTvw`g+h^Hi1&lwc;c8MnySlmZTS@f!d48Zl&`DBmXfY@@H6vo z__hA|$XEA>?z+#e>Rdjil%*6n=*&H&q$k0b+lYKB>0H3}B4&4SLMM)oQ@bCK9_8jSxmlQQ9Z4A_+Anoag9_=d8ymU=XfKj+rt|cyU4(b9Yh3`u726f&u3V+D_s0ijk9j;KVX1#_X>Ec z;_d?4^mY526HnjLjJ!sa8-Y-b5#@@H2%+6z=MC8t@;2{Vvuk5Ji(3m135xp0USQQu z@QA}OxAMkb2@#aR>Z$C_kFRYwvRlN`1wzNN9nWGDvW{3>6{)A;a1;A1HIO^5P>)7N z#4z68P~M4TE#1Vkr4n zYFfco?ls}h6*t|FQ8?}TXm&W4XHu+R0YUVpmOo=QD#JZz(anuSHm5APpHx2eGYf@T zq9F2@bc4rBZ8eG*F3QI+3hlLow%19<31@`^>^U-F|E<$+mj??BO|7FK zL_j(m^;$SQLcI7OF83zy-M$mA73z75Yep_w#MyI28~SKv-P#!9>Dtw^#XV9kxvMEo zhCCmd_*gF$FF+-DItMyV#hx6|ZzE+4!grp{ay0z9zlch*{OaF5T-Aa{AmNZ6cM%rhGNb#Jr~{2jqvS>VCe84pyWv1SEFR zqafOJ`$-)xKa?<1E7*(aw$~Q*+-~3o>vkQ>+4FE@?Eu;~BXHA^V+n(8Ki;$j+vDJT zPrKHp2D++o1mo7c(bh!zwoa;R5v{JO3QTca0p!%^}!`P*eOC z9t*g=G994?>7bVybYdhJ&OPV6%^~>iKwIx909aUVEFKZB%u8 za;V>Mt9qN-y6Eow+4707I$yW^8bGP`e^(uoruJ%kcA5BJ<7MO>w5!;r8!N=y@fwI` z#okVVdfZor-NF=X+A^h(r^e>;t%8qxI0O2i{;-sP1L@&+TEq!MWw8~lHBBorVO7Ly zPT_Z&8el-#;LYu5VBTF?$l4ma^F;lKPJT0mnG+EtG0;@%6l`1o@XjXyYqVFyRQ>+( zYt+qc;G)_ep+(&Gi8PO!4iZTFEj+?C_nn?TJc7?yT z010cFK10foZ6;xl-M!~5Tc%EJ98;o+$|cuW{@yzJlQf1nt9(q z6I*+tUT8g`S#h1NU}Z^IW5%#_^b%aEb})h*X*NarxXJm};iEUx{5mM2Dr;7C3EIT! zV)1g|XFI{@b>{Sz*^ne_HJG%-pfomfoWmzbS}=b3cl?05wi0^z^|??CLM`0w*xUa8 zre8n4@~jz~Ik3Ba2l=kj(RIh(vG^_cNcbz5v-3HvL^wxpwKuQva_SGiIhVaBDo{@Gy!7v2P(tr!3R)qghk0Gs6mmW1;|o zzB110Nb#zXUbWjqPCP>aoxVAVtOXUb(Lfe*?mkxj$8%2(Tq;D85x}RXh3+L{{OM7( zBBMN@`hz}$URjNa30Sb(_(LsiwQ&8;c3;Ajx@xZ$J1oCKSA{eVxV`d(q}^n~3#Hply>>Qy9{(8N2JB6bWr1sr0$t zmCxJ=-KM?k)0&n_&-XESj!3n|ES1-w@pt!O&;ALKYIwLMCYyV zRSq=kzrHGEkjs?*&QDqB>Ymw@e(5r>m4@$Z6pepiE8pbxZiAyCENqNg`tU%<9!}b0 zM7QuMsHDxY;FH(^Bj*;!0z%xj8aW6>uOO3_LkBLAqsD#8#jaB?$n>|`r%5aM>M=hDs14{+k?$w2Utp!+ zH}>}$%}TRGdLOCV*%{0XsGHs>(OEa~40qpUrlM}n(Kei7g3bHuD*iyTh}ZKO9hV2+2dNzU0g&>@M2t5`6G^|peZs%5Ci&|^q!Y3&M3^a8nczA z)6|)PMGlSHS(1sl@~M~r%i4MqxmP)n;~ZGN(s9F33dtP-k~FND@YF;iunN~#zWF5rQ49a zqmp&Mh8*N^U*9a{G@Xty=EG?@JD#idBX%2Ap&t7ANKog%=e0YGJuj!U8#k-M+V0*~ z->@Cr)m)hgEx~c@z#0@J3-+mOf?a~Kv>VLSGPun;H@tRyAP~N378_lZq}sV|!{c>m z2q%B!Wo#jozuI2%d5^>~Px-4GmI0P2$MQ~SL0)q8D&43T%_@9>`!oRUl&B~4*s zg#xDR!mOdjmFy!Fv1Q&o^^RjVr8Rd5i;ZMO_nQ?{mVl!v@Pwf^sBl&&o$=|#Im3!W zY9g%}smjVpVWQEUp*3QQ8hion68mK)L2XI-JAj~btQ0Bi6B?kD!|n%@i-N#teQfue z&w6e&PGDwVu$SxS@_zP)u#Zs|ukR<3@b@dY{K5KbbyiZaepOHcg4Z%Wl?Z-Ldg=O9&+qLSa#GMD5Vovhu@wNudo$%y{Rx?RO;g37%%{}g(MLVmj8`?vxeiB2;DiZgWgw{X+W;f zDzPf+J5in}HXdc=c5ACK`%&F%t-cnyKBMs30jvBEd?H z#r=D8Tz`+C18!h%9}sQBQ54c5!4croW^E473Yx}2xx_pj3=PSrW8+Wm+4@dw8vVXyhwg}1a;*fr~x~%hPfJYi3 zag;aj;z>UbRrGcISW0bS7K$MzOASGmih%vgn7NN`2S`wk+(=Da0SC(HaIh~92H6MB zN>GG;bOCz@I-wevRH^2@f@VP9_BT4??YVj zGiZA%B)J_fLEV$cN1h5|Ttbv8q%W)`>R|h4CN*&*{3;GJ4tsgAf~m+gapEYe6hjwP z$Ho=*bhig@=TnV+5D%n(m?dfmus@XveFVdxaouoQ$gefKF4a@BiiX+}jTSK|Trfz` zK+Lr5m1x3mH3W3&UqI<3zk}VwImIy!QoSaCm)&h3Rv$twJ|(BSPx=p+ci1%5P*22G zHPFVabc$wvXpoCMT1k&pIH_7GE47}Mznd+?HERZm1zseN9%?@U>&Qb{UU`jqcPTwM z8=rO2r9#tE!l=w=z>BpGYIJ1PtP)Ypk6+k2Np1#-5BHC4Y!&^bQE)i@NsacQt?{B* zd}<>DCoy}AvF1x#T`Oyt|Sk_}BbAsb>SsO}w z=q4(DK?HS;ET>1>3js^@v^0*IV&`=zGfh?399H%Gk z;$79|Lj9|yF+j3J4P1-77pX-04+0yXhvXFO>eI;&iTGQ6kRY; zgJD^#Z{M}->)IA=+7ieG>4U>VxQ6l_RwH=*H@-l#?e-O=b^+tuGZz$@^92K>GFtr( zg}VLWYSWR|vPv>Gdn#Z2HkZJ=?{`=q49v#e2Z+-3(^8T#H=lSklwFGCD7dRbS4b00 z*T~S<8X5QBs(m56Ag)x4uT$?nNlk77JDm80EbrQ9g2{ef-9`epwO(0MkVw);nZd>I zttXk3c}DEzVEJY^P{|Cx8WsfrJEumW2mvhwJ(?g#dvy#;7izD-)2$Gp6=hbSYd#dj zI?EiQ3h#acXmG%dGB`*;z!-$8H6P=*s}EqD@sIbsy+lojImm0cQDZ2FER?PiFmSix z@azST&p*I-##2uB;}m`YK}5Pq#Z)x_ifQR3ZB+Zx`5xw&ND3T^F3oA@Q2$JYi7ZEy zsO5Dh2a*T73eCV560SDu6x&~0NoH8h(ZYSRV78XHSr?hAz5woIqG{aPC_^I*Af#R0K3&OjGKzG2aZ$#plLL;S9q1DDLkF-}#s=AYA z_EE_hH06)M*3lb!mo0VzzRV%?17=FCe}^0%&O?K~$**H0_u0SRA6N zlW;Tdt@KySf=(d)j5hQuI*EDExtKw*P--&$RAVFLSf3E2)5({WOkc>GRcW|l7RfK*Z6tq1KgIIchj`d(wZm@Li_&P4z;y%X zi9A2{hF4lj2V)&I8{aktFHCtHEBCjt12hp9qL)|f7A+*Tl&7ex=*m)cNG9G9 z;a+*8XX!?rsXvq5*6G67?Acy+(fFZeFtX-0bYpR&*j6&(<=OFaq#ouq2QSxbWW#!O zkjX2;;xQ>^E& zdxG&fm4x&h#+jus+S@k$HUNB(-Lb(*?GCYaxlV>B!s|#-jS%d$Dt4vm-wuLE{0;LU zkVq-_g|BaUwf*K-9$EEVVwmfJeLi=GLAgS}81KO=lno}## ziPHb+#U5CMPZ%ynQ_gKgEJF~Q5QJq#NI?hs@|^BVJ2w>3ZMjV$xW|&v569{n(Ozd$ ztNDS-xfXv8!4}Lx0NQpX%u-vILZH4W6wiDV~#Kgb+6rINj$$Bogr@ z2%^L>Hwt;ePB-tj&GqA;%iI*Yx#9DPn~S#HK@+$(Xa9`YUU>vtNN&!Gy7bJ`a$(?8 zYqeP$Qi0DUkwii;rSi-tlSqvj(k)v^&r3_QohXrwG|Yu>&Kg^yA;NMur;Vp$i?{V_ zz`ne^;xEIJn4Jz5FK~J3v^4oaUfq6nB+Doc)eGF z4BZq z-UI49MwcWkBQ)cPGf9<)?`~=)+sA=`ZJmBmJrOGk-{&UuS86xG%7)ipyEG_5dRISL zm6Rd~TvJML)wIm(ae4$L2Txvlkz&ghwbIlH4i4Vk884;7v^}JP_uj!2#qbJ}TC`a! zxsuWtxrJ?Yg!JXl3^(Caj7ZI2Ni*FqdF)2f zvCj**GOWeatfZvwGF(R;QD@bnu&&iKx=EyL-ebio01Ce4Q!p-4CE*%yu(0;l)#anL zl^K2}=?J((tYc(=#9W2}?O(D90oiviP&<*yYPup<-7Y}-)KQBF$jdbKTpJ8?G|`Iw+%WR$aV@2` z4wlocZZyH}64ogWj4UoHdmdnL`AtPX^>_%d31d9{Tth0hpQ0x?4ZJLHFKjfBOic`_ zT{o+jZu5513ZT#~=6nAayaFVn*5O!=Tu3aH32Y5I**_v(CNpD)Rd^h9nFPW_QA@jJ zMV!n~_H60m13Fw*cyXtvnLA`0%CUq$sHC3j*z8)~xg03n;M7{_F`4I~ZKRPG*L-4% zoOYq_;NBw++icpjYE#TtXGd=uqDX~z7DcDNq@yiFUYJ#5V)^4)f}*+B0w#6Xu9pOK z+~#6@8O?(%ZYaxVP(Nt9eYP~&3KXzoxIp>}QoF6!D$Ipm)FW?!*L+hiedn)MU7@5O z-WCqwb5ilwwxs4GPLTM&uV5<~I(WD_Im+3M66IZp1JYpX#AWz~5W1v#^Mek@N#WPw z1{QWwO^-aZB~+gBGI_Q_H$TMG3cLTq1`(%;v% zM+z}MlA4shA*(F_60mr05!+mFx~=eMJ5cKgz_gl+NfcSBI+l4vK@J)fZMdM%q(aCX zmtASc0`wf4C)|InAh}luzXxGklg{pwg)B_F>_SW)%`bL9% zfm76nIo5FJR-u~5&cx2gq>^_0gPx4ZB;%!G2ofLo1`!C-FZfudJ!PYABYH>48RBKN z{8D;H#r?#|6eY<1N{rQQak`m;sjL|HoAASkt!Q{uTMWj7#6CmSYs3RbvJ)UwfxDp> z5Wt7Fz`CF(af2{v=n|oRS`H;WekKD)U}Erh)7Fdy;wmm(U8f5 z5!HE-l_Yet1Gz7QnxP-i^8ZzL6jB2|=wecLBvK>3IoQXG1+Iso$5!%zyQ>@ns`Ub! zt4s_~&D$be5*;#7Mbv8cE2z}Ps-!x>0jawzYUD#2*{ri4(g-fOwCVzlI8ay98kuRe%%Vb^&A?MZ^Pkxdgq2Cfo$MaIPo} z^;>!klNcZy4>tPrM@W`bIK${&oDE1Bv<|J~m{ID!BAk&UcXWk+vx`9c& z0ZE*Gt?&%^brAJh1VMap3z2R9ED|453JqS84W%hMq#%M1F7C~C9Z|G-Ta9g+0IJAN zqtK#K_f9l7L1Z3{5R9J<#qEqBLG*CXeN+g3WDqVk2&)tFNE)yvF#xlz{)pYeV_X>o-q$>!hc*1oUf>#1pi@oK?&w$FIP=Ev|P|Ke0 zL3Lsu%bUW{ei!B0EI*L{Ak4HyeLhp=?w^8uD9FjffUhgUJ#)TxM0u-#D=dS2y|p|c z-0cDZ;PJe@LL1NzvHoW;D!ahYx44qqxM?#|OR<@_MG_S6A!7Xpm4Llt#}P)(_Y)Qgn7}+TEp2`ZikKVJ zq;?ZbH`GdUSAnCZnRjp`=e}x-*cB`z>&@C-7z**_!=r9F(d#=UPFxlUpO zT|q^|a>7KcuTpkL4LqF=Co*Nw@Ui{_-B)!pP5N78&fU)^{B4JbGe}-<#sw|E!U66eL~J$xA6;0GxL5&Y6PAqt*;kj!L>f>e<%$)7(X>A441)j4hUi7sbB-JmnXIVc8X)vrgXU?fVmTTof*046x}z`Ul#(y!+Q6TF zb)^%3V#2Ki{tJX(hk4^9s`pAPke&!xbYR1)PSy5d^3)#lQ>(5>7uU(I@4rE7{ehRL zCKeSR+QZ#k>)4j!g9AA(w0bUpfD};~k^RW(*CqQ+^R^UQYB$K66m8QvG zKE+M5GV{GHV=c%Qo*_C%PH-f3oTD=1TbU!LCqpdL^h?wvfEb2Bhc#i*;q=a&*U;GO zWHXqjDxPsTT-dWWg~nb)5Vc|F{N0q4mAB?F$O34d6OX)X_STsuH@+cenjIt1sJ$Y~ zqwli#Xk50~U}IJ4u?4xuH9qbSdty$Irr!MBqU_#9QUK0}sT(ZRMc~4C5nd@xh@C zEP6uUFf10AbCv)RlZ?tLo6x}L(^_&FH1TLD5}%gHfJ4Hkwz?s`b`&fs%ifmy^92tR zNw8~se5%!p;mx=ftQwh;x(;Dv|JP_^3L+Wgbp~tJ^fB>~no<>dUz%~7v1~PVmv)9n zXUy~6*fzprMb2SP3|m*(#a5V4W4o;eC|R^QL&w5OtZ-=GGxou<)vStjy!1Tg@e88I zmP2uOJSSDcm=N$FS8jHY;SwK1#I9tm+{*Rj<+hVa)RA}?CKU~ zH0YP!2&nKA2Eenj!b|wD(ipABz}XI$$-L&}a73=aji}w%o{muEq-7*x#%5T9{b8dt zs%OzD$RGJSmZk2*5wv{aX4LqpDUO^lXrLS8yk07xwKC05cN z)HKNP0}*rQYK*1=MHOfI%_umnKrw(9Mg?AzY^@-5#QnwBg5yTx+NwDe$HH@C@4vxz z=+qq+fVw%|&NxDr`E}zg@RXT|fb|mb(Cc_AV)9vtg&rH4j%7-`uJz`Tfx%9Ghq_QUdFPU}SGOgZyK0RKfhmGbXzH@Ae%GZo^ltQi?Yf{%!{3Vn{UekOfPv3YzC~YaDtF3{S9Wo*zw1*{g9Kb)@E9=l4YRK5;8|@ z!#uK*wLIGG9SsIFDs5GO>dh;l3hx5r9J7D2v&F)2Fjt zA&|tkI(ZfvCNZnQ0wPLE#7`jxAn28A9}$7g7wc1ANj5diMncuE)|L;hwLVWD8Go7l zVa9apHyN^{o9Z13f8R`JpIoxnTL3wP^h|;^AJZ(V;XiF5jN$za-q%^z=o+k44iF-5 zrgC^DEH##mt=KtluuT0o$~Dw#IiXIu8uvhShUAPa5^c&!`m9Gl_>`^zi3vsZTsQJF|*k2&20V&{^ukQfmqnkytl)j&7L zT#_VBi%E&@4})}asST)32%h@W`UZ==$#u6HO6PhLoZ}=AOhFp~0C)WvrV;nfX6{anEeL11pYn78RaKrp+!RI-IY1;bz%-SW;xys_7d5=g+F^ zu5JVJ^aw0olb~H5DoKs4v1`o^7oUFuO9V7~K#GDQSlch>4Pz6RsJxW^o9)BT14u@K zIHND~1h`#d@TeM-D2>eVN>X7~F-^F6XNTPH`qe2ft-6ZS*;sV$xz^hA`7{qHcmw9& z-vH;dA``zWry%CvHv3>W{S!Bl;5_Fur=^+q8PzrmGq~7#ZS4WQ(qO(OP+(YvqHu;| z^a_Lq0FK=Qq1msZQX_f}GavsepCNLQNqVm8l!%#AO9z#-gmUr3`EPztpM4x|8_LN3 zH*Y>j=@yNYrt8!TS*Fge&4Y!FLQ(AXd( zg>b$0!y2|xm0J#mXGD7N&*QYOAOV zT2S}X<34}l@;wk;NoH2wcF&k*J}=E!=`iP&O?Ae0+oJF3v{=iz_IQjB=}u^q#xv=v zwboPOk8seohS*XZ2f~2(acUA>VP;sOFneb?H`FHZq*bU6S!2yZ64tIJG%9!wi>wX| zZ-_O!q(c81lgB)rsIsvm4QI-$7l#0UK!CsTRi?33h=9E7TzEMw4s!QBSBsFOI&cfR z0OWpL)i1VvXI$m`JJ+(n*wQV&N^YF@u+O1DaXYM^Z zZ*laY`0_=ywS$CEO!>lkWiPffqSvMKjkY^$=fTNaeMrP~yj?fTUrpA=GP4qN4zrg; zbhfdveTkb#Q%y^rj7amXly9Y)hy_-Gbkc8@H_pdb=56cC7Y-fqAcf1$xX8tw|=*6hf^<_jLyN z;2BnZz!FdR^_#-R30cXoB$aSZ{)6=P!b3vds5-93<+_%Ge>o6)Go$b{Jl3%tivI0v z4azu@FJ^~DZTa8yq`r(tevb{VZ_0ZwN<(OPgB4?&g}_Ufa&m>K__p_&zACa^yuk}> zSsf%bsty>~-+2wV!j{r7U{!#18Go!P*TyVzvlFDe;yCxAN2Te|ckJx>dsFXDE_{gY zyuxm4FfZBXr&#OuELD1M2d=eX)qW>_pEQ{+rWn+LbW&M^?>uy6!h z*`1vDuV93Xf{D1JVe#q<1E96gp4h(<(HG3FmW8wtA@*pJn4usFg1Yw)$OU$(8l52Y zDfIz|9`-^H9eAsVxpDg5rg6+V0MlSd%_@WDtrBuR#Qcy5}{&z$`7mn){t#%Ik$7vy% z|J~@Nxv#1OO^(T3$4bId-Rt4WWk;h^@1V&mvd7_i0kZIH}UIE*&;{$%60fsl+Uz44q4mgg-`A*brOl%d~$^PApiAhNXz5hKF-BgE$q zGC2Y-ty=dOqcf4BVW$AG#{X7|_P$>(#puTcqL8X=A**=L^T@m8zo{3UgZ_ckpyeKg zvXQ6K(I8mk8*pTOCf5i;sol^-YDh^+a+4*ZQ}3uMy+5DM-N7`HHZX)$4aMOw(_?OC zDIz%CHwI$BA#D__xsDS=oBhR_Q6L000k!4A*I6n!Vk8n$K*)aYFqwU#Rj##CmJ6F+0<(3pKsO z_nS>cp#S*5G~{iU;Y5}^`U`Q~fFsA-@W@H|Y^6YaFn|*2*OWp^pw<4!x8qD%K&7W9 zBs3FY@Y;_aERN*OQ4=@n26&R4f^Sv=xXE5odPx&jFCX=Cy$1otla*tSLF$n4iWsvp z3Z4>h^lDp_3KYbIi$Q^QQ3$f4a{u)|vrgcT%;ZXiE3N+B5m|NCFTFAJXiT8)(qQP= zJARQiB(R4Dj6h=%3BBCYqY{4*eeh z6hQ00AN`@9CxS1-FA%9Z1q5o10rS>YyzE!;Nu7pW`+3)gj!1(eA0_S>dJDbClswbq#sc0o z#$oFy_;L)GTmgmq5Xd<8D`7HS|5msY>?-NvNbx6MoIgE(x})It0`(RDEBL?K zDsTULx$Ll$1R@*Eni@AH(r_R2DLex^`WL6QU7vh${_OneiC*)wQ@wi6GXHG$^pphZ zM>Zgul`++@&0h*rrte-Bsfi&shu8}?fo_G?v%9hU7OtOnWA7!98*nM`lKg5T3-+si zq<3(7ZRdTa?*S4Vid&jlb71G+${;(M?UMG6r}mA7FKEKW>V*_bz8Z*sKYsNeSFhe( zzIpZb=Hm5pa^t_M$R- zgZ%}|80`$}08+6i7=;#B_p~dOIvnGgWSoierniz#V`E;x!_L9OKIF16iE0H+n3|L8 zDmN8$gK2nL(tW=!%lti&1$n=yS0HZZYOeML$hM`qkb9# zxHMUHJK({B&-MnO-K-f>Hr3ocjWX|7Kk|H!z0t4jvjI$`x#XfJ*Rz@FX&%KuA>)xp zM1ahiYnMEc7xIp$y($eC#8~A!b6QB5u}+~)X*r0C)z<|DC_P8jU$8Z% zOlw4O*2eO$nN3P8*|Crg;Y-eO*Qqcj3)6}IFXCaW02^2S6H**cJh(b&e-_mo41q_Hv3vbWX^zBjm`PyS)|;dEQ? z$?3iK@6`ME+WYs$`?vCcSEqkG?cTbfT)#~92`M`pb@Td~7_p?3a0$Vhs!aJI!<4(1 zKS4b4X+ zIK8m%*q~`Bk4Z~v_cyMkkEy8f)8!tS_~JT*_d(NN#P~8g$8WvzH>T5WIl2$4yko^3 zXZPp#vd|LmOtZroayNAjZ7t4!8PgR+kDG5^e)F7Mq^VKZD#eynhN_X86Pokyu)xU~ zx#h#{%k`*u`o{;YWh8TyMn>-THRR($?+I9%sg!0=TNZ>5wyGnDcP_ReuEoeObFark zrN127r6>@si$k%bV7GW)3B~4U0Jt+}J`( zYr;&>e+1iWLu4P2h6UH#znL=^DY5gX07O$VeIWkm$RIhCudnShSnEG!c_z2Eh>R-% zRH@F(1p(*!A{QIXn=K_-F;Ul>_26HZwA9z_nw8{VU7QoV)&W@)YRJ1GmrJZ`TbJT) z6epreRU2jt;gw94q{>xUFxu7k7g# z1p#cqP9<+-HC#XS=sWLOOf*+Ey|vy`D8uaHO*$s-LxGKMfOUoX^PCA&n-qzr|5yiD zBQzZ6ZhMbw;oz)p*KW}y#Xx#YKknY*YwSp_38KM!)v>v8CH1}8dxcLZ+2~*N1%T?+ zrgM_!GdaW6->S*cf;!RM*LtuhOJk`dJ8LPBl@xuXT<1sJ^} zVT$~(lkYCx{q^m|n^&irqV&RKcC~dM6o2O4XW9qUkGidcv-8#`0JO=jLsPm&?bW1-bbCn!M!d)^+&rSJw3M zTfLJg(W+*~AcX~l2HUyW)dpf8a;d@zvvzOBS~e8o#ks+xU($QUaJdV;T>WR(XMcz> zK`m8(>P6Rf+HoS9D#A{0Wkv2O2Ji!=nUfoyKPRt*s<66G+4{_& zeFE~+To%Kk*4k~uO|i1QM93>lcpGp;9kCDPuaC~i(TAnd{}JU<9nHz*CeL`nOD(C0 zB`XTVmIX5}1Yz=}FP=OKQUt$`W-cu}r;^-jxtZN*jtIRWUC~4DbT#u1LWEosx#3<{ zFt{Wk+gOxk3{td1x&qn3QTK{PQ}2g{ND@q**P{bLFPB7OiODll4HGL5aOEBff?RCv z*EI$9kPr$inm?VV?CxBp^ywK)0WoF3fzwzgRpjXD(VSdc`RL4Fw|g!KaVZNo5lGU0 zazsvK0W(UdAQ_{krKlk^TPy*48f>Q-Q$mMCmC+#)ItJN64*P~}*i%#{abhWpgc&n5 zJ`&s=#mds;-ZkwcACbV5XrnAbzkC_LPxcM$AI<+>F>?9R5_qqLf6CQuaHA~K)jOp* z@yW%8{wM``_0_eR*6B~QX+g>}WX7JkJq^m;0XsJRu_~CBOQHlF_jFsh(Z7?Iw4^BG z00j!=5Icsb#g_#%&w*DwA>{yhA_20$dLz>jLWTE|CE0)0Gkk%+lZ}Q(-b%s3!2|CY zO`%=r8UFM8f|o2vjiOkPM)9G}y^RIg7wlqu9~T79=m!ih3NwpLE{H+)hgik8?=D9z z;@fwZb`AQUa=9~om|~bvKFst-3aYi7`gKc2ty#HAgbU7|?#Q^2(q$0ps}J*!Ay*!{ z=eJx01528>SU(Nv1>4NC9znrcEyHdR*gJWzpzSd`ZY2K7@d_GrZus1$k0uxLv9 z-1f^D#6UpTPGE1a3!9;?t|^;F@?nT4Xr>hIHRB}0)J(<7)47!nv55n0!@<;Ow8-Nm zo*f$~InjSR2ixlD9BAgC+_PrIj=*GJ>=lX*lwXE!~$dRBNv7i z0c?H7p;B3qBmyYSxb@jL?DCKx!p|gdu_C85e1Yo??tjG2aoy}Ee$=GAHOnM5w+o@vD5*XVo+TJh!4#M$j3p-G2A8`aNFMN`km%(UpIaq(jIdYxqHjD z1o{*%WubAiXyOtp6WE0FO0i;&pZ-)fcZku!WZ$u#IxPKL`;INUcGmpt{J~*@gA!OX z$Iprbom-9VCV%?9jm^P{$xr!|qj?`Co_8dt8BD3zFQG!;)p(@WPiq8}K8q@>tXI%f zLX=VgCm!Zf9y-%>!^I~<4Tfb*E3)h@KXQ@w?|RFm3C}hKNb{RT04CCODOXvzZaXKX--A+EX1 zp)GOm7GQ&oP<)UTeq}oz6n~R+s;PdaCn-({q?Ws(Vs-(Ab;tq{MQtNIK;}a~xu${%{UX41@)rk&NFm zvQ&R6WR=%_C=hle3uE1|%8D{Vw{2MBxe9DM87+i_S{4xO^#M@z3L%!R zm9&>}rLhzGTJAaHI#OF>zCv88#13TapBFZuA-#r)Qrq&~iv)*c2Gg3+7J-U5F?T49 zpghIO${Z7fVd^)%5%K!m}*RdP4iD2A-vR(hh1ToP(_>ZhwObKC651f{LO)Wh0d zNfeK>Bq&Z!Ubuui8$fg^N>(gs!cKh)1IyQJ!-_z&Y8wn`bKcfeq^3n|!U`ZH>_f>4 zeHJgSFG*VPJ60%i@>M8x-ECC$RSa`m)vmAJ$q71dc-<{IB1 zK@&*fR#t`adD-eOi7d3v-IoYk52*UF(ke*dH4w(OTnC5g?iU(eyIlICk%Mb8Wmjx( z9h7ar#a(v~%BeEeL~xZ4b(g{69UH1xARW1?4~?{|*wjEtXr5<5=X=y%3z@R3YLRiZ zz8-D#?|K=7ubj$0)D)LEZ$pReUooLL$R(|cJNg)g_NRSeB;@pE%+%@8e|lTb|GVK7 zPc19zH@+3Q{|-rLmZ^3<3`_RIGP(u{D(u2y8rMi@cLYK=P^y??J7PB8iK56xV)CgV zQdj_Igl3ujPF*h`Jlleq3lKuP$=UMRJ6raTqPChL(g0ifWT^C2JuD-#3~T>zyfJx% z7iC%SMOB7Tc1$lbe-49Z+1SeVj6kl@symRenc3jPyJisvW%_eC#@FYalPUR?F&k2OWI`gva{wLcU6J6%9q ze9U18IJ|bfV?U)AtjoY=%3!xWSwWQO^m2vFcSH?vu|1q>XekRqot$cuQ|@$WD=4bdpf|P*RoF^5)^G8$rgSLYw$r{mNqk6>k>=9Yr`Re}oJ-R=`GGtj0Bb;j z=~(k|okD`mywN%6c_4E+VAjI{!VVQeB4<=**dbhbK8%5;utPz)YboX>X zcei7NTZ&kO2^GI)mK$fFCLmrdqfarq&^|e! z!3WvXjW{5G_%cS+^{(BAXU-DabRRZEizVmcr=R3+RiSv8V z7BRL?(bszlMYSm8Djt~yyCR9vGLeO#NX9oDiU*{w(KhOqAL75oIp7b3@7O^=KYS?JfiJspIKDi%nWm%pMZYm5Hst z#T~h$g&JNl882p~{EjObg)YH9^z9MTm%i_v=*f3Bsok3IU@w};)JeeO%mq#O}*Qu-sH`WO*=uy|{829GXb$HuRGyy%oc zK;SZa$s$qNAAG~;kS!2R(GUnBQ>ORZ0>U{p z%)jVDjkv2P2FR(F6W@TC@aV@!qKg3VAK7ay;{=^iLtHYRH%4>BR>Gg1rj1l7CFq%^ zoXnDBGND;SxjfR8VgPww*}G~Tff|MM7g72HvuLKm25*|@8Bd@fK4_|N%nogCpr-E! zzNt2(QX0vnFwX>`nj7OLcyDX@Lva)?0AN#pACd{K-A>k~Ajl}S>ZPhc*)S_W`8Nnr zF2TL;#7*TYqVHTr!L#_=XbV`oL#TuaMEfY$=aZCm_LOjjnVXGxi{fra?(rsj^D0Ao zL23V zut2{y=`bnFSP2;{%$4oO=$VbY)5!UQAm1u7y0?>zD!aR#aE+F%NV*ktICU4}>J*p8`vdbJlw7&e z%jj=e8C_w~Zn*~eLy7vs4 zQtkyS?5()^?!_75b2hjCD+tnpv237}D~z#sDIb9FumsQE9E7R&$-;gAd%bY|<{xIi z_qrYY;l1WKg?=ESMTmLBkLz~L+umQv^g3tB-1O&pkBTv2qAUg~Y+cFJ>!gfxF4*5} z^aLE4v)d-LWMVL2*=EBES(>K0OK?x=JvjkuAr?P4Bc+r;cYqcKVO|KrK5zwz+gYH1?H3&Te9J* zq_=ENuC>77hc4hr3Ia9ySxPEV@^JK#e$#y)w!s%tE|)}AiDrso2V-Mt0@t;NszIs4!Qu9m@@Cmm;Y}AvoB9f;MkK0?M`~Y6z_b#;+q-!jz)L z_6&NO@zTabV?=FuuKENvRABe8f+bR@vZxX`DM}WFyk)|rCTMPO6A}W6jS9e%u@S5g zx*;DOLwDo^rlYyPwFUJ*o!V%bJ%&Q0WwkYg4wzT z8o|v*R{{~cx#h!#X-%_NtW)E|EH}XH;1~v>Qe&*>Rt)eEl{9=wbKjxG zk8#mqKe#tCjq~TVTDR3N*U1-{+@rv}g%3oz9X0@^Z1=b&YkIf8rMZ?~XRgx>Olbz6 zn>~$ji9|#c+A-487L%`Pho({2(QctuQXhaafbq2SGCH)>2L4P@FT-O8#0BFTkX3QzkV+7c$FO0o!G? z|2Q;gR55{$E9m+3~RDhzE;%x>F&j|5g9oY``wtGL(EF2(4DL8AfSNREw^73f9;I$ehaNXC=pW3OtQXn-Wa*tMi8S-fr>cE&IoNVTOa(z)ukcQCqfD& zd-!ae;D(m8PUQx{IO)UYlgDycuPl=lkGM!RKLEA}h746D35&c+#%$n`(kz%eaWf1x;@6X^|=@a6n6r9ox=O zYV!pwaeN$iMeAecqAh3oi$1CU7I#hCGU}<5eHC*(!0Jti16q}GLrb231&}X2@7l(7 zsj#sWaaR;X5fGNlVVOJGEv-i4eS0H4?82k*_8{(*+&*C`L?e%x3EyA{33$>A7I`^0 zTB)PZ26t&|LO7SHH{qRwjFS?6MZO0gx(5l(t63)R6)EqfvwETQ7k6AsLygo~OC2z9 zwRowIG8yV4?jQoVGkw?ZS(fSlhA8%c={vA`z7m+-&y_Vzn&g_3w2*loD=2APKW(ob z!a#~UrXU!|CjErkGx%f01S@C>j!#p;lP?L~nQG%I?7a~_8laQe9_dI*?H#*;0OmxE z0rv7n9v2jbIwjwl7VwV0xV-Eh`#_u!gf;}U51{A9mcgQ#%Ma10B&dD{iJ#+j;H-o^UVQY$=vXb&l)x7kkn}0_G9l;C7 zY^%o)fA-cbmk2lBGF9i7dcn_rK)!n*+F{k@cD-DZcPx>2tUv^K^&zK1aVfs0Jd5=R zSyZW=LeQ$N8J{rH3+V{DIqr?w08{%*!!&`RdBN_utm5{;q%l+_N*Jm%y84>0TxVAs zstlnwMH;USV@NSAE7h#El0ZPi1G>1~**}KFiOCqT>}q@oxyIY^nXzQg>{zm2hAjD1 zlQWv*0g_S44()l38SVJv!l;EdvxtWK;kSoef(@@KnU^weXu{|#(Wtt3zM+Dy*vx^Q z)yYA@?#uod>8v0w6@R)PKi~WD&X##--4Nu=@rUd33U_Zvqe!m(-WZu&Zg^QT8|Lh8 z2E?bNk@fZbVQ2z4P%y5UeQ>=tJp}+X?Ry@`!e|dIY74?QIkrN8yFD{Gc-C<+dIu7S z)FkXf!dPl1|M`)dQIBH@fVi3z4ScpQn#z21!7}pqiWOU;&Y$K#v2ET3A{F~f-P9gIB2vt8v$~-q;GSKMNS-1 z?W_FJ*UAdd271xNH45?vmRL7O0vY6b+hqLW$v$>39QMo(*amm(jU?b(wjHSGygRU} zl@w~tbKAZb?zzDR@*U0i7%cKvh|2DzI3sUmssHl|GDt?^r5-OOQ*UJn?|+m-!sQ=u zM8?0z!LNo{UQlUc8|UpWXuuOZRt<7l>foawIQh7(4qV?%RIb^com7jS|8XM`Th z3pFPfL~*gom=D96^}NqTgVi(AdeYx3en&Hg`HWP!o^NuwBY(zI3vBYLMFN-GcXZ3P zM`vxi6bSUuRn6fYl(>vkG(EqFM(lXqSuO4-nM!{4rF+{g>6Glid zYiMb0!}dy&S4A$B>HE~aGjWuYyNbp(JruK+-35UZQj@kE1@WR;qs=&&pzxIOEz9nW zh=OiINl|Tcy>4)-m{@*%jBz){rXGu_T9qpO7_d?{7Xs<TOK{H$8J3>cqgfNiOw1nTi2cGw}SEOI8$kc8hH{<%;@D zDjA+MeCZ6&t~-WvOm>`!(52*x;k8vIE!i3NLtJLIZAvr~+Q^hIO$6b!=L|i=ViKkv zgkOX?3as7UhjXIhe`|u5M4v=Enz1{Ma(^5RL{dHyCmrgrWJgC|FQ{&B3!pVhZAKK3 zv4i)@E8hV9E!|5Rsb94+pv`x2 zp+&I-zKifeVui!mB(BcFsMSYG-Z4t{3{EHTK*YBOA6fPmv1m4AO`7J%ulb^2bI*o^^9qy!Lg=+BZXGUZ9;?`~AA>o*@IUHK@)4bAo5XNZa2mmt1a~&yov|FEjH( z5H4>>H*?c0b1m;lD#<;AKqM`vLS})A>6qiK9+L7BxycK;;WkR%;>5uPc(W?l5~+E@ zSta!G)}3a)enCoFtXQex5K(OwELha^VS9{?Zryq8dUrNfv47)DLf%=3kbwj;&VAf=oNEsV8p#_us#bZ?<+^zteY`_p->+d!7y#c~j&>13C?@ zB*L^isuOJ3czJ)%(-E5ik2g2Ls=j&?h~5RlQzlA%xr?xzb^_2&$t#W}`2aSwXW1;c zsOvk{2QcIX()Y?5`V}j&Xc!cIkmZ^vzNxa33MMO+ZG-6aF%Mywxn}GGUW7O%>mF}1 zTS(jDi9N7EwZR6c)ET$wx#PU9Xde5(5>sp?&#D#gEsgb2`{CEEy(2FLI;+_lV7Mc9Y3 zpuvwPa z+)O4Ehu&eaZB}A~OD4b#x`i)r2Q&RvK&*5P9cga_X9x@z!HLHlJ#=l}+BU;*PWXbCdBkyXWP4 zW|Ec9L4Z8}CH!a5+JH*cMVgwNO{|w{sx!>J7O!WJ+H*!Kp8j#jDmF^w88lw(<1T#c z_U}t5*c;b2$UNJ!XqX?3kmXq_`mGr~77TM|4L7x=?Cb}OaXmoYiaTx-IZ06o(SK-B zq8TlzV&#xIt|_0@?`Z1b-x2dC@ISGX&;gQRYR{6{AB>nM8c>$9a2sRW4Tb5i4X=f7 z>pd=5GEn_E3j58BzWw&{<=Ab~?^r<){@vWF2#+7QdHCNeUj>5_|88>(tix70c#BpE zt4*daxw?&O!?p8T^J@L+#VcZdjnrJa?Sw;y!hW!_t{8t{Pn%W3YFA3i)$Jjzz4kXf zw)F+ud7^F&6U4WztjN77FmCje8=gNWuY{@!RyPG|eP#%UBefRt3Tw@)=T<)n0v>6? zY?~>Aupb&5f**^rm$q%9Lt4h;*h_O-l%Q_KBd2@Cl0xqXaVU0PE^R_j(+|#-dnl}A zyBczu3;7uMwd>15&Qo@Gu2TB+45kp1t=n8DRpjXD(VSfK4bNziVL>og70o^uHi^r^ zO$374o*a=AS-^}EDoDm`VKGBtwx|^h=xiU0?2!jIf9L5a(d8JQ_gJcLUb=c&JF5oN zl0Ut8H7DO1RmQMoq-uKo8mHljkGZi?SN8x5L5w~h<^e5Tf7j>mQ*2n+o-nMLAV%#i zQ}Oa0%X`BOTapw$+_L}jQvV_3qV+ib3O~jiM@{LVy4NGi%CtlNqS=KEqJH9YP zTflBrE&R!vu+0^}GqoHKwnf*|i9TZQa#}U;-Rr~WfwytQEKG|pGlxWOHrS13?Mj}- z6}WVi7I*9qUMDY8e<>_8xqU2UE;G66?!i1_ zBYug&$TK?AJAAlWQ?AEoH09cENWC&A&6n%wY=WPB;SVenws&L5zp-F=v7kR-x%Rqn z<*e9n%>8Ldi(wt9SCV|Z4L($ebXTlj%U=LrJKRSZd4M^-i@ya7q`vYh$SBc_r1Q#A{P+LJRuU}kWHvJ-d z#!p8CPzM#{KEFsGGlkVWjI5fhX?{_a>z7&V2Ln3=8*c>RdZUAgTElqp z=Zh;sE4_QAY14S1WKgg18graSuh6`A!1^qN$5`FNhR-+rSOb=KTJGV;3d1xr8|}vm z`>#GiDE@(*25$5}i|>jHUUFK>VwB&Y@s=~8RH~F4ykTI*Qz+ z{5_uR@xP0fr4*O?e=C~d_Pu=wGhcK2<7d}+%}%sFhPTdMQmKJxFDlH+%7O}|&#mb% zIW|ttV{%d^dCK-kpy>|10<#dCe7s z<_z_thljZ`ZDB!jyV6Q0&;Lr(w0Mp@cxfc5VlZLcwql^dd9hE*ZEkuU-MqL0D2n{< zw@*Nl`{LPep9By8HHOz}T52viAm0E%NNcVU zFQ-khv7dQttW9r2!W4~o{!5VVy-VaU(*yq}jcz-#{rzB; zXmC?H^}dJE`G#qR!A9_{ubVMSXe%S8$oN8QI0ON9XA2<;%SP zS&S*t(R`&9L1X^bP9p8sMn0{(cMW&1)o-uw4u&OSr?if|f?+?1^SiaK;HP!<_UDoV zRT&)H46bIcstgC~ABgfJXpVis7&7$1)f9(<(aO6^k4;7FZ1$!kh5Q?o3XLNo_GvWU zW0R^Z)2AYZ-v-MRR_G76#}-W+nqwPylFEB=Pm46rEIKO90TFSiEM_y&p?(K~>%gb0 z|Hp!truMU`5bf#OVONQ$osyc2Xp&*R;@QOnkx5B2+uD+q_l$+5$tO>qpzZO=mtTHK zpg~y56255zA^6m9o<5zE|99~w6w*&jyIh48t8hkeAUEJ3;C<p_vpbIJE))i1dSvSn9HPxA}m_CcbA>g3;MG zMWxEGDv@Rpg<~75J$kVN0b_ejYKCoR9aVo=pV9j{RK_i9c<<|$w!0sTfND(V`aV%X zXjRG?l%co5)^qZMAW&EN4C)@o6f!pw)b4z|)AwsB$JDz0L3F@#v)9+pIEogwt5)n( z1>LYSO>J#~<7Tx$#aDtAXpf0&8bw<@+h7a-f-a7K7)6yi&4+ihdfhIUwZ~+%TuuN- zA~&X|h)4;%MAnR^w)?aei}IYMb3$I|%edqTEu)$l>bFytR$8t$IihPuWRWr~kG~{| z2hi}K$m<21fa@79cv;Y5i^7XNJ58*xKSAUU!>q7dBV=EN#eFimp&j7a^@#qGE&7D7 zf{3?!xzw1ZcF-^f8Fm}_3HnEoUWks`WJ7XU-dFFR4hFREw zDXpd-0xQoDsq=!BShwd0*C8ZiWE{bxk)WY*K~-YagFe9Qjul%2I|}aj28a9h-yG9{ zGa4ibw6E;w!8ro%=vGC0^f-HrPMndwI7dOeWrk3Nx^a$$2mr_S&dAE_(n z1NG%RpgG=ty*VGPJLeC^dmojM<@SLb7N>wLt1osZJ7^HF+s z9?-RO&%T`p;QD^OJNN0{Iktc2UL8F5>fyQ19_-P_^UzM7d!F*6b@P0*ex8rq(ep8S zdd7G490Sty|9pVHo_pMj(Jvg<-7_w397m_lYmsm-s~JJlk*QK`w8|dU)Kg<4A#1E? zsP=KkHuAblTW);(9%`;#RP2nTQXH4Yypzimi$7I)yo&FI@M~tedl)C`RHLsrj-}4@ zj`EByGOOo9s1w@wAo$E?cjd}tZ|+;r42jp!=G2YP#O%-ZGSzA z2cm=g4b4Zol-zrM9|uAgvMOzMEdD)WyPyJkDXsVIfsNB6RFh^ZD})|2V_(h)qsdx} zU^5VMB5_I!oK~`YsTb>fWo^DQV%La=tX{x`@w>PfD`kR=Vj+TzDZB_^YlAa)&HfDQ zkCo$gJ5*;}>FM6Gt=e5hEI+Ypty9PK^)R6xmGHhXWLjb!Q@YZd=gbp@WqJ^W zN4JdvoAHL+OqgFV8$^}V{ zf*BQ1H435&z8VOy+QX9E6O82rT#W+i1!r9@ftYj_!5szDkEwHln?h&27o|1MFIGo% z4Mc5?0vn3(YDIC4f*XkVifIOJM{Ee#&a$FH!rCKV55v)n4)yAe77fj8lt8^iEMA`jIvu##C-x)P{+b50zvFl#`>O4b%+%}qxy+pxrVCTw0E&mhKH+{akk zBh?`sds-4>t?PT50SAjnM6T{5mzuyaw%|l0*qE{uI6j`K(Y|Wip13k|4M{$7fwCZ) zMuYNW#ceEK7svd^nY~6K{vpJbve?S;Jw*nPw<}jY%L0xXi@M2n8?d|S>KHj zf1)+|{gtPkseP?kWgW*6%{Kf4Eu~3WsF`ow95mi3Wb%iB&^)3-2Ji=p9koZ3y1%pB z&sbH_8WTTj9MYDTNuPr=j!+?FiNP>uXo?@@NF*+d*w4!@yzH}4vg5w7RyBV(C}_pN zu)Cy^HQUh9a+V%n>yx=M;$>VJnv`aZ?Yp=Wl z(rB{;6+0WN{1C4ifU}@!VtMPN`=W6iW|8Tz#8~Ufs8b4A zV$Q7CGgi3@Sz@epWgN^0NNwAJ8I|fb3gnO_2F_{}#9>Q}Z(SJ$I$(*3&X?PT8I?ab z+9DOU#E>M)tz3_vmnFs%Z!FhrSY^f=I{XT6)-+Ss-i@eh(H_Q`-tYIA^=ycju2Eu_~r0N!WcNI$~~0+=Uxb!LFgH*6z|?U@6p|2UGd z)w40sAlir|RZ%ceX4{V}nY;aq42O|SbL6xvSs|X2|NqGk|N4*F=^sx{{`1M~fB)CNp8PO}fBfq7 zkEj2z|KndzPft$%^Dl4ybaVCUzxe4t{!>(&Tl`=DIQdWZ>c1j^oSy#ie~p}ILoeW> zoXKK_NBSHhAV-0-+kM!*u33NGo@JbZKtM^*N?S*) zTIAwPf37-E?Vl-5O=$#6*hP}aN|ZP9mPG`hv}_4G5TM0W+qMv6O91P`F1W`HVWU;b zwHm(%j@eiJ=8hs|sry4DiWzEB>ebH41u0mI}|$XZIwwxp0HEu%4LcZoh$*-z?^L_$Q7tt__C zo<#sxK&Zc(>3I9Vy^9cUT9R36TYLWQiPnD;bK5nKfo5Z+Bw2jifaXq1e^!l8Jo`0S z%c@ZGm*ad&r6f<`O*AQ4{kkL>qpBoNpO6g~RT*2Nz*AX%`|A>C;7I|T!6SOK z1{YnuDeQ^FH2s%*bnpv8e4w_$1zT}d7Grh3twbP6zpHv54zrb6hOaCJlQ> zQ>n;TP^#=D1EJmLwY~e&ZsEJzs<*SjDEmM+dB)WI_IF@wGk5B|SfuCgUR}I=^J>0H z4{0(vvWeDlxPx7zxpSD;>8;qxiriDIvuHZs-SGT5c_mZ@Ep}+x`plr52Baje4<31C zHa9F$euQp#hqjh27GXUbO{i;mPf|(lA>`mr>u%uHSJs;19pOCNJcME6Y|) zJGX;-aPkdZjX+0u+!-C_FyHRIiByndoJhHpL6=0UWwVr7^IE|#c&j|bDHa12^*#|a zdf=Aq?S_M4VlTQI{CfS@KiW18_gPO0Rt}_qv|Fv4S3p1a9F4Rp*G!Z=LCdZh4mk8l z7r9)bR*|w4dfya0Wr{fZR3l*!q~O`d*xgw7O^gF~oa^>uJ7veZW1}3cqv+hAA#OkC z;wXLb7!<|^frMXn2qaB6d>}vilOU19DjCRCy??b#bzhB4B{XA~-$ZF;=GmrVc5Qz2 z`ZEuxA$>i<|KgB`2$}Xf3a0+xO^wLIdTd*KDAQiilHJpIo^77D+Xt(oX<9I)ECn@K z5?tcN)g}28f;zlUc_B*_KVQlX<-tIOb`Ragi*`{ygN<3tm4ibJprtIl62Fz!Ta7;X z6Sjul5s5yZ*3jccrdne1>~c2603&zwA;laoFdY}kKok>-gA>q;Wic&$xT^C5P^=1D zW?#z{w~fFw`ryV=9;`<@$w8mJ9HdnCO3+0X_Ovz;+Py-)&ghCT_#fvfSNgx?`ua6^ zQYv9|f#5EHZ^6)Cmr;sD;qbePgl<&hZ|@z#MQR&@diCSX6NAC()TQ5Gu+f5%n>9~v zuRQ08Ec9=}`zhLpGvd7{JTuF>8e=%{-mx6Z3TD@4m*8(yDVgxWM!^84U-O!>vCcD=HSI3UJYgYcSOBYY%k9D{aw4K#=ExLQ`@#} z+dj2z+qP}nwr#slexGli{BI_+=Cv}Ji^)p%#m;Vv*ztgNcUAZ!Gx`>8U6amXP3>Ak zX(@2i7D_zsue2TsQr;PRW5XC=bi0c?c;icSl#QwC698SWbCQx`gzL+6Lz+k1K2=GB z8gydw*#86h^+RY&@pqI=7xERwsiU#8fJ)EB*h!-b0$$Qe>O{%uuK_g_Q^}l)UVtj1 zeP!LU5Nc%vm8?lZg(0Y1&ezLoZ%$%rmmh@Cmdjj?4tkS5L6l(%%30B%aX-|(e}yvc zy7HjUm?~O@P-E%KR7%8jOav@F&J&%pah@Io)0~a%d!LW!adzy~@Xa(U{tPj~;Iq>d zKC`En@Jw4SJYEnWk+LTfl-FI``+XlYeb?Y)bqdBet%l_Dg(%#|-DbBpVMi{|d6az5ET)J^iGqu<*<+&RiFkB{F6fZH~ z#Wx!vhn5bqf_+t7p|NyaIqzz>W|iHTQJZ7qTlAv1JY@3*-98Z(O89dA)%|#UIw%*l zk+F&bJpeY$&>nIg1G>1p*&SXcm4JarA7oZqL+wzR9kF&ptcXXimKK{#uzB$3ZA(PC zZFD=}2@}%Su_;SC$A+`zj;(}n%bONH8QM}YuGW7KOlpz5>s6yE5yz@-e*pcE1lqy- z3|tVnTp?$S-M}4GkH+o4PVU~zdE3W!3!#97*-(q?)(|pRtrQf2*KDioxA=(C&h6CA zl_1xnZ`cNrlOxWA5+x_6*72VQR~vT z@)$KyAvjV@Abu<*o~CYP)ZW?DEV<12t;?)9oWfxh#!OeTfOld#d+Y_B!A+%2{T)L- z%${JW{iaJZT3lpPwjKmEw$=P$S<0vgbkc2xRn@omtBx+Dfezo#am##qsSAdK1Yc_0iR}yNp$}|4 zdlN!5?UgypG5vcans|8QY8`#Z?qOggk9u;yucb-{X<{R~8dXA)NbUIWJ$$(sQ%>aB zh0Z+N>husuu&dW)+jlqt7Eqi6i5uisDQK(SZmNQ@`8C)ZZ}w;=cOu6;<+k#n(<^wB zGo8zuUAUZ0cZ8dHo6sJ&;5Lg6%Jqa?rBQ)Ib~%CKpQkc=l0z%5ms~<$y3p|lPlOHG zRrY+AjZKZD6nwib0qqO|O2)t)iylo>30<PAksSOl36Amyk8i$#F0|D zY)v++!6K`Dm8Kj<*ESwA#{S$NC#+hRB`Y1AnaLO}mGLUwvO2jxC-r^FQq)cer8SR7Sc*nd;?dCMOLoeQ7rf5nKpW*6Pf9*ZejkRXAAo(;_PqK z>t)v~4nD{>ZyI$zxc<{xj6<@P@@H&%w+7bC6p%-ml?Yv5@x(%hc37O`-SNXcYqzIJ z|15(7IH)AcE+0(f5z~XJ?qx8~*t}2m`Ve6#43MiFQ%2h%R4y4-kRlnqwb(9;#@({~ z%9erIruy0;{)H4-2u|qgN16UMg5%S#gSSrU&y~x|uY)(wV?*f0V5vsS?>fGt`7{8x zM+OO9ooCxD-VmvTzi*q}*A(rS;~%w@(!|hLFg3kzh%k^DGtiIw6?Cb7WKVJRQ4+Qauc0rM%)8&0+ooHc`cN0Gh{sgk42dQNXxlrRX2~SrEFh*s3_w zUDl|Lf$tok1j&*i2Mfc1zN&E%BDQaH4c`{e-H=0hR_!wv9&$MltquH)ghjN@eUlZV zPIK#1_!85%jyRv}q>t~LcZ97eMx{a*BJVV|+S*M^1%4`QRZe78u!<^{5(7<=mB+c; z2T~H4cI(M}!Jc``xrExCIenAi?el#u-AW&|oQ)2OP}q=^CiVKd(q{)yJi7~VF>9{Z zJpUMnM1@(_f9*WP zz6mp)^I)=(`xNhyToX+4aBjOYCW7!?#S<3Q)hpyY>q@kS6vljc#!g=u!9h?1&MSVN^3FReMaS)@k z48CjFdasgl?n@5U=Z^W+XA2?#eY6~z>g$81NsCslxv1q@qBpiJ97dz4I_1p+&bOp2 z4-)sOoLUCIvfJVHzF0}tYVFC>=Lz{%)BIz_`dX*@E8zMmUge6wiY@ULx_((YrHM~y zMd7%6kap}Y3TY%Nw!c!Fsfr6BEqAJUeOKeaD{9Kbdbc?C$nXfWG*UKVby z$wEDGW~;5u^v9}@5fL~3ZBwL!%{#OfH6Pt z@NdQsO=*m;U|yuu3>jux$h#Y@c(xx?J@W5gs1g~zz@dYzGo3dJ5*%CzD7U}s-5=_k zXK804l?@}>!uHaNp>L9Zf^dV26*KiH>sfC#vaCBb#NIB-l?oUN8C!5^gDshBp$|||Dl{@ zYF{?*imcjKPm|A!%z?R`M|3SJFQRJd0~YV+v?pR7)_rf5X|m_^QN%Ua{voPq{%iA&LF zIrnyp_@=&3mTdd?Q|D2Va{x4F#DRoBYs-fK`YVpu6FWp_K1 z;Oy6XMH56#Saq+QSs-CyLzQPb9}} zYQmagFJq15T)*vl%OR9!PY)~zUt9h)-7LrQ!xa#duUaB$C9ZruDQcmYO^Y0qb4p~;~oW6)+v^il-Me2qM$hji~H)zqhwmeGJull;DK zWh>1hap#VsqHIblZ8ED0hd z;(d{GNFAyino4ITUN-QCYI=R%*rvhV$=F8f`@oU6!h^94R&T<-v>_u!e{HCI=K0II z|Cy5FK{#=<&%dr~eIMG=Uyz*(z~T@r086YBA;C;rd6>cOQ~dGHc1+q$vIL(t(V51FNUXQw$n@5iZfKf&Dp{Ok~7B$J!o zQpx4i`xCkEvZ&4-0dJouI5)ky<8Q@I=crFUF}B%~^#e(zzzA~DC`>euWRWwwH#t+i zUaY=`p5LxKOi~d~Th1%ThAeW%{Y49U@0Wj0z71J3zYG5GxXYT)tcNB*i+kEf2j%`8 zq(nY({8K1bwA}$%LZmn8vwsA$6j&yR2FGDe8I9=#%q&NtHKL(7(YDf1r8g#I2>wh1?#1yAugz^oi6z1TNLHYRK%(LWI6K%-CZ?VjH}LtK>fZ( zIa{-SfwPs8>X@fr{jKKfIuSLPZmh`Da5@;HS!wXAIEs!v$Zbx6#>R-Pw*dP1^!;;@g+X z)qeZ5b*rBSt>=RVRjwm0A3DZKo#%iF-DT-dMAy*+5Dn!0UMSQ8e$VtfO@{%r>oMDZ z&JyaPEoZb#O9Rep<0Reh>VV|UNOmJqQ!JMKkv$>A*z>>JgdEoD^??zfviwZJ5phg1 zTVO)5V>}Mb3lin{B|B4F{#b zVGxz%BYkCcfyhI{43vvbjuhiCoCg~)no5UXfyXVX||ofw-7BDd>eR(d-6a%ZV~Mxo&lfBc&bV5erW7!7cAf^=Ula+R>{q$TXs zQ!yxXX2(ktqwkVL=Ve${zq&z9_t5Sp=ef-Wk_xwIk1YbgIJ-SdR2$j4?W!;PnBDf15g)68n%{9K(d8WjbmOt&kRT&zTu&B+Gf$ z=dQ#YE{aIL95LS9_}3MJUI~-vmXbZq{>+QF`t-ckKvP}tLjb3;UmzS(Fiw=dtjt-D;5Wol9uGUvLfa1lL=>~A0{P zwxy0%KqLDdMgcG3@m7|vfU9ZWNzth5bN2oP(iyil6~sL$mc(!(G)tGZP31Py8P`Oj z{!FX3vrPC0F9s;3!O?T>J%M6$q;1oY)j3OhvIY3rx&Y5xi^=ubxn1J-IPPUQ!&gFE z+VSk~9vA#}_T{ZU)jS@pl&PJ0>h>BWDnPn~(TP6kQ-Rxd&69TAJcwM{XJlY5%+ra0 z4oul~wt#{;{Q1HNd-%$g%3Ry7rX%ANj9yFTCucEKB=&K&96}bf=albkJ^4(b zygQZ@e|@Y(peYwjuhuyifrer|#TekcjD)Pj$TrEoJin0qeO z(fokH+M=qCv3WYZ0q9jrxrn}HO)IDsOXzKibY`sxV5gF9>4(wE5E0EoGfZ(rrx3rl}N3WU@ zw@+Kt^Ea-0P*V~>76q@kY<5qRZ!kou2L(eKFLE9DXtSuZvQOeBB=;-o;#RdEH)XsX z^L!cwM@lZl_7As^nDoKsMGk^_0VORwLDX7rq~l;xj;?Qi4KtvFSNeNVRe5*=B)=fY zxAlz$jQ~@#E zt!M6baphzLvxaFYkzva!kI|GTs~wgf9d2FOLf8#{LO4vMA3G45y#Qu?qds#8_6N1h z+ooHZivw>Ev_mmq1Fcn(v8Ja2*R}Fh`+D{0ckuK)Q16gRhp?z8E|_HKDwKSVyRh-7y5T)Fhdna2U>=QI+GdhZTfG z?L_){>r3gm6tn7To#hI(v$T}&qm+?`vUT(_26|bPDf(W!s)0e>Ca9n3@NXUKKYHm? z@FRFdG&##~=xH4WDCj-c7)+w9`JiGFa0j*wLc}XoX{nY2==C~7@a*kGm#=L=|APIM z!@A)_FRPgLxhYqO6#J@NOb76Ma_;x&p(=F1+^YWkJE}2(kyeIro&>~y%LPs%9P>DT zBAhT8!2In#O(h1X%Z>@(kVd^Q%3`-LyWWNe)PLGEg=NY(u(JRhUn~9qLfUDBMNhuo z@aX*NJds>z4NHl3EM~tzg58kb`edCXFm_)yv}%*#lqZVUJI6n;OFHC*^HgxqVZSqQ z;J7C2873{Q8=e}P7W>4s$H1(G9xp*@3Kp)fNw+3R+`tPKtMDYg@ehB?u$Of0KPUl; zVzN8e6URcc2@#9&h8)($PHdoPe;EsKpmKVP*wQxFW);#+zgD<{;nq56lY02iyER-C z6Ol%1sWn_V$=2Ao>~X0lZ@i7GLT@cdb|EQ#k)y2F3-jU-cIWkGZ3F}mGVe#atv$vB z|K{d;s>!FD!bMZJ1ceVI;a-$jdZoZh2vW~=Aqy(1)` z?F{4m{tV;%@vs@j1WaTA^8|k~0L9Z$DIusgYOm;&v!Q0!4JH>P%_G9joePjP-C}ow zzGFIsqy8LwHh+eVuMro0v=#q|R^b^0?XB67808w^2%UeTc#A%^=6^qB- zrQAWK@u{M{gy57v8q=w2F{zluY#2^KA%PUbVl?84F6f1qb+DLYSwWV1w{wNy?%FF= zz~@HxvHnp9Yn$SQAoo#Ri|AHe!OE?s$oA}H1=Jl=O`Tgr4;68WFkDhu!_~B- zBZ{Ad>q;e%2MsyF(8zeH9wOLV+q$T`(1t)=bS)!WzqJs2SN2mm6u~*FCz^L*Y*cLG zHM?xNIm`izB)U;2dfvHzg2lQQ3)nHLIP%Kmn#hP&x5%5sLq&AX9IIEHS5)r@ z*J*uhcy**XZ`?YTy3UHmGHqH^@G4C&_WUZp#B$1vZQrl4X5P-Tf>x4P;Ksz4=yNS^ z?8lvF2VF#Tvv72|8#^tPtsPz2$h;-O`~q1%?!V zDWjc8tkv4nd8I^Z9x;|~7^wr*%{(q~zi6!G2 z@-cNrw6L?#f6NZ6Rtvg=Stmm5jG*=7xg!T1i+2vfa0@`%Z4zLwAn9EYQ3_YtFq|Dg zxffh&#^w3nNP%JWRdjHeq~RUwR_WsS)x2l8%(k?hlaItcr-LBMIiB6#m*nf$AgTvE zV@m;9ptti+kIn-xF#VH6G(@bD_Re~O@|f+Iz%?hL7MjFOad=L}z2(h+f+FK`*&@cW zK&|`^lEv(F4lhBY98DW2^`!?}#^%wYI&3CVnkEj|5oT2uMQFBB8rx^QQW`XMV78PF z6ZIp}!8EJamBkWTpyVeC|E3l0IQ3Q=^;cZV?( z!}x-lNoMjc|1|v&?>uq7nRccn-M{ZHBmkP|$#&|oSP0RjSeG5EE_vMueT@#<;0r^R zRt@5t;YKV=G%pW!m}n6=>u(W8Dnc9?tHEGm!Jau^ufJKGYN*qUc7};emX4w_GiCb> z9EJMQGiBod7LGzPvn1UAADpCRDImPQ8O)SvP6Mh*KCwzyGgXqCjwHHRPpOyQYU@UD zN(tGTFMy*tV2Wzt*$iW|d#=v69XcE~Wf^>dj@Qc9xaI~u9?gvKsAOzXdxy@+tK-ZQ z6Y^IYCKPGY+h}5vveb<%DdgMnN{9_W{V3k21j+}4oYF@KH3}uwiQiUD;@gJ8u(%GO zZof0oqMsd2KCcif#1){=e93~xGsD0tC^V?fitSHC~O_ z`JIAU7lCY*TfVZFUcQ<#S8c4swFWi020^?GaH}`D9(&!kF~6qp=kX(NnUKsebD=_) z{VN39zi@OvyyL|q6p^F!`;m6iLrRS*Z=d)o+6X7L3QnL|2ttPB`XbtBq&<>!J$>SN zhQ=`;#kz^L+g$7;u+TW!J@4f><&jm>Hq~g?kX~kHWw_J}IiVGA5@vdmw?B z37p7%|Cu$Rog1Y8$WXT1ND_kJO(X}xNFsvZbvPTsNCE=&kWT%{eL7#@9pTtlf1}Tc zNK9whoj@A`8VepJRJ4BS{XrkF!a*UECmw0#DFc%SuN!hl;xLoK{H$MutH~kUz}3VG zU}6XW)*)pTYH?05C##8FP#lQp3(@52%sO*ud=ND;z@VEUkXH|2XQ_=zGt+*NVQM6U zaS7}f>6{k`qz}|!3pHMnrUT_7OYQGV(}ox{as)`z1&*80x5IIc95I?lkDJV4B~I?; zj+h|kUNSJw7Uo9`m%}4yqZ#B8398+QCQU3QZz7FtVg{{Od%z!C!v$>rEj=t{qKF#X zchCBYj&H|yP{=J}dSFUw^%qD%Dj8kNg6ErabVmt!VA;q|K~f_lQZph)Fhab_|A!Ti ziIWO}n3{P{aQ}@oZ6iGXYG;3FoVZ%-RnP zWTTEb%G?j#x8!AJ8Nf6LR(iWsBFp|w&f+qV&bfb8ugvH&03BWaraqb$5i2u1T5@2d zbYI`bfQEkwP`a-#e^bBWQo64nlan4Eg&0xxnu_whIWSTp&zh}l9$11tHCt3OxCml$ zQ7#)@f%0(H@;oO&Y?&K+vWmh84Do2?)ipt1RU?($MI*Nj+wf`?<#sMt< zJ9r>t|G)TAo_b5lR`{QN3EJ%cF8#kJN@v@H%znXf?K3-idu;9pp`SBMAYI#oAXfi1 zEyI<)T>`Ch^Z1H73r7LkJWFA!AeRL)-=;NFmdo0k*|6m>@;m_ZV_h^7eIA15ciDC< z$zfNL%PJ?I=}YrCkOsf;z+gdZ%Y@mI0l)DWRo?S zp`0$-oYA{DypdC-wi|hZ!(( z1i*3gw!^uO95JS8Lwp-ugfl%4-H$GznEkgaA3a@0A-@yLu;>@5a>ZBx+*rd|e!v&b z=YwAf2CmBss8tODqpx(8zuF>1TVT<2o$7_77F?mVQyqmQTd7id9;SBbLv_VcEhL-1 z@LcG|9f?R&JQEGEuwuI|+Q~8msKWyzOu>+TJbdU8rK-VbS%Qs$(D8N6u}?xxzq?h* z_mjl0^rW3$7u!nz99$qvhcJmS6 zVnm@=wZ*qedrdRF@Iq;$A9Wv8cIx%!;)CNwu_Tj%OQlnA;I5=$c?RRmu7&KW##yDH zR&CWAi~=sRxBdXko{RKfpB*lKvxVPN{<`jGC2tCD#o05 zz=@Ta9C9?(TWKnGMkz>@b4SmL@p_llt4Yv>pI|T)O|9pq@N;iCxp!j=&;|pI&bH$Jvay^a0tvyh%7U>0IGN~BQ6f^mVk~q8 zeb_%F8w2rUvWldXlTRDoe%Cy)Pugs(4Bl$>x2A~8musxauNN=RZA6ogg;BJ2)D#m+ z6OTsJkvk))k@bP?i?82CdjmoJz}zH2l2pSN5wk6f!?}k>``>@U#yvvhK+{%7PJbl_ zL1P{c8+rfuAJlVPm{X;o&)WU&>&_qf3Zxce*vv*y!yn~Njhy~?~q zSD)ab3wSM&h~jHoJ(t=|b?5`?PkpD+J$100m5Jp^SgH2p{jZ0iQd8v7%hww+0reMQcSkpN1T7PW{utrRKNJ5X8a z?RwN*j*(SToDjWw@~rTEM~B!}PfTxjYf3kyIwfBVk*FeCw3^05uI zM0M81Eh;+Y>kTWGU#0Ld?|zeBr5XWUN+lFJGE=JBIAk7Il=P}GKUfF&*h?)9Nkc~v z^rayufZd=`EZmqjH*oJtEOsp^J8xEWpt$qL_p`t7s`I#g9HNgHm=1g5A_m%pkLNQw z)J7a`XVk4Uih2-mpT7U%e=nvHp>NalU&)G&}F5`X=M_865F;V;E{ z>s4T8KK3K8>vg@eaWV{%Wi;APx_rgi7gIYsTQg(FU9*L6<{vd^YrMXLsqr$_AY^2` zF2tALOr!gP1)6cx$8F~*217&!6G^Hd^SlLet%FfECPJy^!0G6pOcc;9`0Mocb-2He zFiZaXyqFuqDyFB(xBHK;bZ3_O_H^r0TTQ0GkS`8#*u!g z3k%mY6p~?-2_7ZM=fDuKJYxTky@>_MxjsA}Dj|#oiJ~0%hJ|o*jd^F^kbaoM$?l`4 z1C{y;R|vJ=zX`bwSZ-m&fgi$wda-8hsZNyIK_%g@xhYDh&7Z}C;$Q) zYdUjAc*+taQHUw3qgM>f(@7)e3)CwL`i~VW%>ygb{Uef9fid90zI|d$ew&1j^Ya~r z%mLR!f|<(WV{*_1g*jm+-q|+RfK&EMQmg|Y##YFeyeDrDw!vU+*z8!|XI4tE&)M3V zDq%`#SL?ep(=3o^bZ)~8bNLRx2+Sy%_CV^!BchcwTn0=T+s+ORl)=*Cs8of(&j+5-S_hym=M6^`6okW6i?3tF9pv5k2_lqTYZ?Q!dUylbU0zkSwYP3hYesw4=eOxQ|By2l=!Wq#B z+2*n7VT76nY=tERzcGSoImb03A9w=uODqJRRr%^@K5^XIq8D^eIp|Q;rmWI&^h0So zWE;fl3*y%E8tXl5Mi%EYe?PkS^g;E{Mb2x1Cb9YPbq84w(CgVtMl>iQu>OD#kh9fd z?^9(dTxqqQNLV_pHVZWy8B77I!>2JZqrrBA#)BW^%}eFHv-Y(knmYenwy0A@!2&} zGN_p5QT3pd9hr3TflP*4G}X9RS-4A3Ien=sX`<+&N#0lPg5wcqu;0w)Nv5_kX(-Hz zn}@{m;gIT!tQg455@MmrDt;92pI;X!4VXKEq#%m1UugSH%%(*)jNtRymS2UjCx! zqeugW7bSH4(1;o&LyM!H;qZdNV0lFP!8Jr;pAqOBgZM_hwJ2mJd?I}hw)S1DxwRuv z2rk!Fk=-Vvq|}O~DDZgC>4dY6S)f11mVEAsZ*$(eMD``UP-{YyzR#^VnA&fX=r0$* z98E`R_GTbwC=_>SdK*@`(k_+C{d7jxe(ij&*fqz`aa6iFI^}YS9`@fg@uQ-&7Dx52 zQQR%}$>e+W`yLty?UM9EBBqW2&igHn?ucNp1`X%yG z+iMG`r)H4)o}pPq91?Xb;4a}?`o9d0jC3#o!NN)TB*f~{sO@V0oN~Tbeq412$|@=q zI%}|d_R5sCgd$y0^9a_#GjD4mSvge^t10Ieb}C(XO1;$ff2irWw40-+e@i+z^KAB* z(S#)zH`bXWAb2%6xHw&cBQ>5M0y4=+%zFgVRF>ejpu>{iyDq^%rC^R|MU81}DOC`f zabJ!od1-{^Wy|DPVkTswP?(8?;vpdk#e~m9-p3BUUln#fM= zz$6i`1HdQba!-b!lKP`6OUIi%_+{n54&cYDHq2yI;7-6E%W_BpFDza-i1aQD(iMU8 z+1X#`0l3XYa#6X_GiQ5}K7})9ZCOR%X2d*n3-y14#b!-TSH07@3}uu13?EN6m~-1S zpmpigR&NE*xpr%z0$(Ub&o+*Px(=hL=wjxhN?**x>(iil$+ts8!LPQmO3FjVwxi|5 zdARcWkR1j!{ct9ul}W_B=0OzsoaL3_&hk!}l$h+gSWZF1!>)gnAgPgRU9RK}k^j zVL{eJiZTkU3i_bG1?^i6DKE~N!gFar6#%WXc=6NEO4#K|HQ zFz!z$0m78{xLv`#d!^Y_*C>b&D?KkUO-Am{^0_}fcOpQH+G-X7Q-01u_&t3WQwn*H z<&*a1{8&&n=H>o6IqZzp^X__g|8Z~??&bb?z^8pq_jD~`%nlLPF;QlOwxK9gw74Jo z<*6*x`AFcZ-|A_EFp9Ckrfwe~iz8jdzy#57t@#JL23SoPR!a>!&lSJuu0jCOY-^;tlW}<5XO)&#N`u&x#ij)7-Bq3nF7;5H z!?O-vUno+V8HcI0xYT|h7{Ya#b&BY+VfbVrrwp*xY+Ptre;E`;O$v_g;R|4{H56N* z_nN3#-?N?o2v|iRt+4x}X)Fwppg&3O6|{^A#Xi1_M%2B$N{L?*(ShzB8Z%_UXH8m1yTFsKi?Yx|AC- z%O`>`@@%%Nt*3NjEWjZ_F0B`0CDOLl-BONh3@1ZU335?k6d7ckbB0qD^Kk6uv=O}` z>mXVg^8GcL4vPh&s56?t`CN7PO+T57OakDD*qK6SSYrCTp1yAAl~_O9lIJha`GE}j zwLB|>mfUGlvSEHy6I#?$I1DO~PoR_xK^@BoEAc|G3G=ixcsyS>Sb`W<{UuXdOo}U< zBYK=@&8P^q3TFTGa&>>aKRj)}E0EiJ^No4?r=O5FpQ*^)0+F0mG{YMQ9#-_rkgpZN z0MLXVtgkzcmZ(8&HbKyrDD_XZ67YicAd03kH*CaV}X< zJ5Xv=Z^ESQ_g8LguoJLL?!KvoON}FPyidN*HGZvO`=67hM3^fl%ZNbC&e9P~B4(aj zB`$Ea$>rM`qpHH^I`?C)F`*lVh^CuvfL6O(5Wx^4wv@{KQ0Qw)bj$2CiZW0~HO){{ zS|5h&`z0dKa@^gXhs3iIgHAb2&%rh~8OrCbjt~lrqZ*FE)Z`Rc`%&77G*(QU&WrF& zhfIxKm6!Mlko&X`=)V8CjKU95b{xUPdnP1hl&dU3t}NLIo0B7)eLb99(EofzJZA5rd zkN4lm4#q$}PzR85lUH3U! zR_H<%n7F0eWC`VgGFX%uYJlp=Y!kMWN2OHr3+1DM0>Zjkfb%|0h+VG{jLCrO2>Tm@ zi@^v)BXjYLpgR*npz>3?!TdN+jHIN2bi7`w$WbuiyWH)-W<%L`+$j?_AeTa>AsMxq zL0O<%&~~+f5#n~mb<6zpSb6g*7z2ESSk;PJ^L;l9++dkhLi+5Lm+FxB=E3`3=lAvX z=TF~|x8)S`KGi5^ze$Su4`FwC5J&_8WBo;3~&)_2Ri|@LuyQw_?kO;n$e5 z{@^YJjUD-Ia;e-sB{%Ah?77IIu6&Fr>jfR2@bwovMe!>4PTgf0KZFJb_If-0%=Y5r zd@ZF9Ud4WR=RJ)4YypAbolzcA&L0s%kag)Fmnj{72CKy@t0y~l94P8cRrsQFcc+Q; zKj)+ZuS7tgJY@)S?@4a3Ro0ani#WpZ^~^)j4_cymQj>O|C{tddkrS;ZjMrB#Sb#-7 zN8%N4;)8}S49L6yYXJ$kyM9AlUY6BTaI8jGh;WdYYav{t(6~XVXbR^Z{1we?lFc{i zVv8+N_@k)uv_vVwpx&p^QcfVAG@ljllHD-}AO2_hZ>faBh_=(Gv#cW>>*!5xk$Sv# zB?wA+Tc4Dpwl@K~{c{)acG?l$S6oeC6+RJVa7to>j82s8ZBsdCe4693DKkPRqZ#IU z-GH||Vw9cENQAHo8l*hxx+XPhwd~zCLmW=AvkYmhpioir>nlJE+M{Lh3{V=`xp`@&=`_Ig;I)=7$V?i+{Q z(%@XD}sI-fYgRKKotnz2}VwxBNvKp(H>6wBKcdb&EbTEAN0y zE)(JeDf73mGEsS*d=hQtnYTAa6-owUQfhjEh}@GTX=`}}CTmIpIcldL?3+|bvEyXu zIuV&ho@%z^!Vt`Tu*GOqrg3h6ESi(F1Yj(urgt$F2W0wW8^}6Ks!{f%`4G-3U`2Jw{pi3ql zy&{C&*9=9VNLXoQ@7?9943tEE6A$)u6rB)R7Z?Q}rq1P15Q^VIbTXR@t{Ih;EbyZ(1yU2k*vXb2$127CYoLhU=~tN(g=ZZw~5$cB48_B$K4q-a+X*T z#wm;(CRJl1es)PT<7*oM;&=AfwSX6>oWCHhm$A-lS3^9?HUUT=O*SMhHoVd@Fey`KbuYqZupboitfvj3rQJ ztbYJ~p=YR@`t|zld}H4uF4g*EDZ4Yy>!pzUmi>bDo*hos7a+8gON61Zq0M+ExO?*x zAZ;}tWi&zaV{UhDv;-xYhBfyUO$dC#R9N2StHR45 z`sdAKgEz^LfJ5nDS)}$xThtV}fH*^O77B@v5QC&Jj?{5@#ozBLJ0D%nspf4xC^aNr z6|9VJ6F(nRVGZgGcdwY9#9d{L_lL8$bV77USl}4;35R}sjyZ^Q;w&Sp3TkF)f_)HR zmgtMpZ}i$?{Bn&cCftX26zjio7w6voZ5%{VRlcnT%C+U|?ZosMQAXQPK-f+JDsL68 zZXbV}*yGIRewIm_Izd8$T{F`}SK&UGjw1sBlT#46j{2RL>T%-4J(-J3k>?pAK?Jl_FfNd>9qkiBUwawm7i; zSiI4%()dAjs$)4mrl(#+EPl_)oRto4fg1MI4gWY&An+rxWa`Uc)ySDWk7uVnnPOhH z7b`=57)8cj_)A7f1v_J{4oQ!3rVm{c=Qq^S$$4(_&s!z zPextkZpSUVSW4b6IXkg7B{)lL^YT>+6@Yw&vPyrfZbb05skYKY=_U=XNWeH{Z|@!? z0{;b@aM&G)d&TIgk0NsvX93FFTb+=wN3L{X1c55=C zbp!b*82Z@nxB$Epnz3Eb6x103C=%wdLVoQA@fT<1F;1__+Kxq{lHaGzNpM@o8i!W9>le#aGZYABNKFk&F3oqqQZ44S9_Q~V`uvqVL#YsZDIwd>jrJp{gl zcmX-pIH1{FO4}PCQn_S(8hC1j!cBoQ%}Ri?lMdPoV5~ES`u8+MNA8_<*Xq-6EW#O! z*#!?qUOC!sCWf!VBZ|L#R*2=TTo`Nlq!7(pQO)pCCW^m2bMYT|82u0C#;I65DZCp! zDKtmTi?RIYI4iV_*-d9=_$c!kE4S$Ay=Lw>+*PSey0be>s z*UpK2{#xoOS&7R3aZYk)>Xph&BnM~vRl|Pgx$LP2aYhQk48UN#yqzt=m+~Q&d6zR7 z!8ad>P5L03e(kZ~4Y>`Ge$_I4uPZ{dX?c}?5GZ3H`z+z$?J6{I`fj1*6_vhVS{}{Z zC$0_#ZPU#H5J%8xVZ2XEBDNoB+~Qho)w(6kcTKT$NO~uqbv(3AN``# zr}Te_PBSHGPLHPU3XwH8Nk9boZqE7vfPB0+Wf-Vzev9l%{PP$&{TtYcg>05m5LTM6 zr!zgnjTMnjoS#(v4}!~YS||f>8<;l|_D`lW1wAhVJ%e;=V@cR`Kml=lGl*P|9tJ3k zhlE)isVNe(iE!51_fQh)rATJgo>&l#j#2wnKM?mDD{Nvtf}~Y##n*)bx?DJjRHf39 zSH|6^Ab#p=m;3NB5l2cSC<^q{GUz$!+978S1-*kFa~A9W|e8_mb~LN|I=9OxaV;3(boJOBDQ9H!0CGivU` z(Ra$ewF`BMn?j0rVilEylQS^-X8(IYzk2{`<#aP%`iA;5)cuJJnIQ!6hWL8S=hZOL zr@%X??z<19XHb<(KO)wq_pAS;IvVD#pxR}C-X*Zo+3&-T3I#2)DKqC$?FtGe_rxB# zU0CJf;M1GW%Xc^nOXpeEdz+ZJn+MA`qnik4S2HMETdnBGbwgX2%32tx)|N}n>6!0x z$x&EnQnp9M&=_@n6;4zsc{^+z_ELkB8F!i=de71;#?zHu-YTZCTx zocTA0i@&#~9S9Cb@n#gp$e(8h?*C)mpPGDux-H=|Z98|`wr$(CZSS;mr)~3}wr$(C zZFkgjqN1x$b#>Rpd$s18GsgG^Ka)V(i7Klorjra$o~eHwrpK`93ZdWWC1a&Fwsw9D zQwA@Fg}BXfOaSMpFj{U7i|?qxo@l;c^dYD+vl5!H^?SnbWB)o&)cy1?yeN*|Y{I>g zoxfy`!sdtv!Z+4S{rqe!0HhpblnC_s)#n;$am+jUsO#b^NGyh0Lx5jouMti_zi5LWt)^ZBh9}%UbfziJLNVnc^XiwZH83y z(F8j^Z{#|5EXis8-Oi7k?|q^jUEOiD^W{ATFoVSUolI*_r* zA+Z=zpa<%!_bp;PGwmx893pH>9R5@s$6T^}_>Y5c5L`Vx_+%J8Kvuh8ZSf{ejZn5X<6 zw#&!_zKv9|-A`iRV4FE~eIO>`R46rgZ*=u`ww8#p5h?!B))-~)_6{8C>Ke>?b6;+r zYBN1;O05vZOqCi$c3m4GN9Jbzk?N~WWnpi4vYkbn2meC z=dfZM*ihV%g)Li$IuKoJ0qj_m7Mi+FTAgwfC)tr`qEJG0k!BD^VIGHG@#dzzM_YTV z&E6w?1#{~+L0SvN>*3(tz~HA{0>LkloKZ~?yKT1H(&_4DIlHz;l5a>@xn=5a_Ek&O z5PMKX8C(9kh}?P3hH)+YEB9u>zj&T07XT}g1=+(S4^dT$$;vEKQ{?aCm^7)${Zw7t`a#~V{}5G z!Ll86CFShosg7%Vc zm>#xQ&(7dkCCM@r(H&UsA!bNU>?O5-0X*<||9a(gz~>D*V)zi}VKMEBzc>p#&Prje4@ z=-pgTmXVUynEx;JI=Scj)`{g6)Vj59rb+VYjop`*RS<#+dmd}yrdCJvHno>fKvCX?456vz;-&7 z*G%BQwgYKRtZ>GC0eBK4p$5=GaQju7&rw2k0joG+E}3nI{L{?2Az2{o;6$%_k92&2 zybS2M7N$}3gecff7!ik=sCTv6Ce3`CJD!11ua-2Q!PV>uvmsVsr_qE1$!ljS8TU`x z1hR+c0hR1Kr`V^rTW--y_ir(dN!BOGe%up;UtQprxC~kMs*6r%*4WFW)!Yt6&AMk) zJ3%l(YSY;Fe->p)M3zWf7fT0P{#ho;W?wQ5TyvJAc#Oqx^0M<-r;3o&A}4>(Sb10z zFiVNC!?mlYKa4ifZuy)I8*eNlKn-9xIMw4N9cAo>%Y3W$Mlaeq8C?pW~+Xs;YCIqWxe zF(UegyZlAWM49m#PlgVI48w23j$IcipQs(EQMwv#sOKQb{Sm)g8ezII1T3EuL{-FQ z2A?SU21^gQ0Ge)!#h~L7p0XBNT;Y9M35P*x7MI#Dw6!ZKzhB@ih8pqOtn^Li(6DdD zhC;=IqByG6=U|zcjM>^1pro!z1-2{ynDj%@6l{(ibsLk}owGX4$?ADhA8bRXCYQL% zEuvWXH5AGWWqmBz#&QsLX#+Q-7LTw`l;?eWUdbLn1dvrXBVVptxrcK!Q0!j8V;Tl- zgG9GEin|(I*aK#_qbxVg*-YHJ+4le>V94Typ4z%;jM(;k@xdY0IoiYY=E?4xl=IB;5!W5DhhJ{7KpEyRuVyDwO zQ_j7`*V#N*-0&2+l)j*!s<& zncT_cr@q?5RHgg@yCVa|M_$6H!a2Qz6 z`9dPy4;&RfzY}_J1J6L44xQtG1%;4K;6f{^DaN$&??7+1??t!X8<6Q8Ml&7@Wd;_F zLcXlEh`~&5W)>w$^)F@y=CxG^8YK8UQ>tLs)n??|WG0Ytr9saK21R(F=eP?7s37wu zWmZ@fR2V3F+-TO(&@q{nx=QT>$s9f|Kj;L8@&ljtUWdXiIm~#Ot8fEvUw7F87!Czk zoL?J%5=H;PF%0&ftW-5`pH#V1dDuyh<&}#$h6IkSBDnn?^a?Y?0=3qQqJQ}n9Ziz7 zJy?`A=GFdseLtRGdSIrVCDI&d+o>;zE3bQeniBI-kM<=`Qms`JiszMl$-i_k!;#HH z3VTyyleYEEF*D`wT@Q79GJu#!FD9wp4aly63wV}}?WpzaSmT~Km+W0(PRG%Knv*phS;?$S{DtX@AW4U_x$bg7QKokc48D4a z48$?0UM=FIXIbacxhxH8ZLfozEC}*Q^_)`Q`nD<)*=X9985L(#n8g~)39C-BdIIR6 z7HV0bb&#_L&&?wn&!X{L8}eA;lqNLfCb}k$y&woE=vjdq{H^i7dWM zMaY)}VwvjGzuhv>^3Ij_Pw3vFhAQCvn$mT{w1PRzWzTY46d5~0nPxTh+|Us0i?|HX z1Aqa=VPvBQw3qz9I7vANt(dhwLi`fp-mww^I&*P)P9}NlE6*4Rf9n8*x)?*DM7*OB zPNb8j#~kqT#Wk37@2F z<|#O5UxeDip=+eLJc5=l+ffBY9;GND!^69E98<5!d^D3$dNiwO#yRLNT8ntO5 z-o74fo;lg=h`|0)L7uFDy1T@=I%d&jv%>E@$qWd}es&fo1IxE9R@Kk&p9=BBSv!<< zhmm!X_ao?;{2R|V#`%{(`(9m&Q|wvnFM4ueJnTs9;PU}&IBbQ;Q*Jh=UJmkc83D|Y zvj-S94j28&CSW}$9o!`h1gM~)I^dAN$?iUDw`j0&*^$ETgjVq~9w5^e0{8*9wJY{% z@R3zActLedEEGvH@0XQp-~5m!e4xXJ?-+u&*!2<7n2fy)gO~{c78(H3XgZ|>m5g0( zUN1v`in=dDv?7<-uF$)8oMe}<0JEwcex@kyI|g|;BOBDb;X_J-Qcj@#fq6}UFb!nI_wBt#{(9*$aWF7EkNaix~zo{k{UAJ1Lu z-O^glsw(crtBEII_PtZ5*Gp7AZjm{-RA+RPYscJ%U z`RB}PB`1BPN!I9&NTuPT>Dso`P}6pdz9dgdw=s)X_-dT3?4G)5>O$LqMjTzw%UtYr zcmH9rnEVtZuBx`P)n^|{GIH@`!EEH6NTq-KkTVxC2>GUQ;J$k}lJbqRx42<_J9YGj z@0ph~Bx=Ais8*F^v<_rK>0$PGE5m1cl1&FSjZ*Tr&$Qu!+b}$4*>Ydk)xUpP#W1U+ z;m;`=$JY3L1?kVEt)$OhX9K_cGvHQzTAEa4N(!Tg2IBj%-^EY+SK*^)U&N**aFguV z#35H(7r$zj39AXL{*;)YC6vx{o2JJGu9<`U4FN~ut42p^)&lWo0u{MK7I5TbE?-4t zEA5Q!##Znkj_1KNy5v`xbjblYxbGRlTKWpbo>cCRetChmvI zt!b4-_y6t{Uql%QG$X{ps{IaxT@1Anmtsb6ks?&-4NL|$RcjwcG| zn^2rZnc#X>0FR3QDTX$!gf{Fwifh|_a-A6+xExN2x* z8KLLlJ8Y%ciPm%982QhRRHuQYAU(_kps!1+r0bE=bKLM1iqkbp82}3#Ho5?5#9f=m!VhRBx6n`c@Uad-uflT z2mEN2tZdax;uT%zJcNj_bIpV6>#-GSq-xH7qV&t6|2)JkSleN{twA$5c|nZJLEL7u zKKU}#SMgM1Od%*)ud~V2eJzS-~}e*lmBONT34|_ z5Ef=%3+c|)cB0JC#4kP(D!o^YW{cZ9B1|tIg-oUnP$DU5Pz2i)B1lbR=zss!`xA76 zGp)x%WL==-6xwKw$}*7p5go83_|Uz5Et4V!KwaoVd-Kv+}vrT7>X~@myc; zY9F;VQmA87P-?VYJa*pb;Fvn?SU>2KoTx`Up z!@_)9tAk_k9+t)HB2}5JhKtIc`uEFxAL`p61Wqe%oWw{-zBawtzT-M1pnX|h43VdEemy$ za>ewi&q~k(WN(tHl3lwIKafa7)Nr3Q&8Y$WyqqQAc@rhQL0y{x-WmwlC~Qa&(p@4Fa<#5xJK)t53f zM7+s8YuH{FOyfG+NYD3LY=sCE7(-ta7PA``*ToE}XJ_`U&Ik0Wim&C)W2XxC;6=(B zVb6BHzt5MChmSSvczkX+U~6|8^z{qhdj<$eQB#EzpuDOCGEmM^QjJ458I^0KQyzFu zo$1GxekoJ%hvk~1i>16_$3&CxC^A!ON=-US*=g|0DpoNt_LZxlRF;-pgZ-U_ zez%u>tmNt|xZQj2rG0`0jk-`9K#Y>^E^4&X6Mv(U{(CZQ!R zx+=w*wNgBqf0m`f&=W3>49JjiCQ>Qd@+tfknR68O*2vrP>ArymJzq!K4s}>l*t;I* zW<)AcZ`CTa-Q{err+6sfV zCc&QN+V?1E^kYERg2+b{4B|$S+dX;vg7WV3g)06xWDDD1%h~(r-4w~%IJ|g0-FpjYihm(KRGH~ipoq!Nnj7|HQCLsUN8&R9JLK_s z^v;?NGicj}^`z3;hKdr)Vc;N3vcK(7_lq6;=G8~Ay7YJNU_1<{goI2{Ol+9dkI@@% zH`JU@Dz{0L37+lFGOkX>DLuq14R&j~OfY?bgtebuZJg(KA` zGRXzmNBN>l31My_MJGZ8gy}c#7AFK7qTGc~q_z`3JTv772Uydd+P`p2*5X)c<)#@J zXkmYYCrU+d4~Pp6;39B8G$3?eOZ_cLw221phkxzF`hV8?dhS>_dRssG@P$+^UI_#h ztxZ9?612^mbt_THv<)-yW&s(0v7%w@bY~?H2wn`)%3IHbBz_;H;(kz0kLZRhcrl<` z2$#?1M&5O-{>7kH%mZ(kRCCUQG{!iZ(xF}s#A3uJ6c>HdR2P6C_GR1k<>4a;QNAmZ zE(nv-G+wn4q+T|CJCZgCJ;V(cUPtmF|GD-vH}Gr%eNPf5jX7@UgAQyLH$eh>$0f1N zGE+^YV0~88tvwB`;I(m!U~dNo6TNi&oa3EA96b~9{d>pY8$e*#DWE%;Wnma-n}l)2 zq%c1@f;*SUNNi1!M+l8c_!PxVvOq(%J92fw(oW6tQ1CN$dP2fUB7FT9)Nj*hW$VLo zmZFC-Pc!%$Q!23^(JpHOGl2JZS4Z?lNG;&QNr~V+;T&g!oM*`buGwclzWL|8M&-DH z(SXrokAHk3x!3`0b{4+jIK<6ZV94kE7FY8Hbe?i%-OF%%yy-q$M#GXteT@)N&<=A1 z<)2CQzAE15N`=}B0Ap1UnV|oj4=672cMGqH{GU5tLnD+G&V1_gl<`qNd&9OI0H?1L zY@lUX1FR!d+Q;X;zoJ5Wm8@k&*`_T6mK7A1CcU(Y*rX72Mbyw&<3f+U(}y^6ZWK(- zMjP8kniQRst?ZffX@^oIu8$Y7PoHrX&^Y*vpr&B@8zlkyW;i{#S%h!DJ5-MmeIX8T zAGoCj4KPsnC-;%xH{j8ahHSG%%@wfzjsEfVe&f!O1cQsjN^ud31W`5rDXAy^h3@_G z?(%NSQPaf?^F6!&7#BW5{Zv*oJ)gEb>#;Li@%lLN$@S`&`g40cp9UHud40v;db+cb zuE78$cn|UCHs4d?8JbdZ$50Dhs#&h<8ZIW=SaCI3z8uS};ZoVhUPJ;(+xAn@VOiwRk zOE#+Em~}tMkxR}~L)E%*tpeiq2(Q!i7&3pEjUpx%GdkdRjytVoQKZj&Vg}B#kVt&_#H%2N%WKmW$Vl^3~@(l>3ZW&jN+dBAnF5*Y=+{j{000VWZ!oBg zYoYeur?dpGLBjlNVOFJNE^Hbk+M4S!Zy7p|y1%syyahM@p?=3LWZFas&_4RpPv9@uGZVCTv00ixbr_v(Pxz#dF2N13h(UKA=8v_g3%#cT- z(1S@O|JdV3YaZKIErixq(Eug1euoh`T z6iopZ6zS;{d#x|F^rl+iG?Kp=mWZvjj8g)@rsrspk~%J)5jONhvLEi7br7`gr&6n& zp@ZaGyVLuNpVIu9d34!0xR9@`)k31cIPz+Z52MH$3;yg?8wT_>^vT%h8J*=81>K&NA$D zpSvH|dO#;GTswa5Y8*qbS{yyi1wtPwpmIgBINjQL4OHBOh+T_ITB+3)I#HbqyYx32 z8)4QFCw&yMySN16`rx^a;!X9SZCsC(Xh%DDXTN> zXasNe=?t4;cl};{4FUbVR*_DrG_#P3$inFWE8ZTa-s38^WiaXyAf8rsB?dgpQHCWu z^oy3t>CPCz?yQ_cvwy;KUE~(iqA-VuI;>TSo7FekO5UAT`KXTBf&!z7x@Qc3IcuhN zWDIf>X|}@b_7YkCjIhZX6}@M9naPC|X9v#fQmagjB#Fwt{oT~fN}Tnv{2Br8F#DPb z_10ON8P$R%cUh2+yTnbA9wbD&piUnjc<~dA{+EMyK89VrMP!)dk+cPDgMg=4>&K@IM=^rOr{I<^>9Ly0Z_)UrV^pLj$d;^yhh&Y7sK zK8`^vMZlSva1tw1y{ogUJmr&sg`3E2&?ykY{0vxg-RUBuPs}I2*nf7`L9Itr22K0E zipcN~tO#THYfU?{iJJ&vs6jJ3ZB<`?*{uF_N7d#Po<%mUAZlv4L^GI1$*+%~nOXDr zZZ<8W0fnKwhbHd_e+~8jX z@K;}iVWf~>;6Y8zX-JE$>q8d;s8#ONPmLoY4eO&>&TzA%%rZpCE6I@E4qq$m{%$Ht zB6tHNvb%k*HMCxAmOH%HRi3s%ZScS( zjs+2RVo@!h&EjZ1K8YZHr?%gCVZ5FuYV|+$sQ_uQ@b%%(no@U*Prx2WU1DbKXzO(E z#p5z8$~pSi-}{fRhm(Wp;rC_zC?_#jUOVShB3GWe>`VY=kfoKrc@{NmJ|2^gb>*fR zhYd*(rjJVdhE8A@-~ry0@MQ%o;3h3*tyv&G1FSC$Y!)LGB>;=g&pVn7$T=U$dR+#n z+=PF+&@)inR)l^Br=LdzYud7fob5hgMtL`ET8&=K&#+?@IRMa~g~8=>E%W%Bp90 zs0vQ|AJo6nqh}(|GN_C1gZ=%xJzT%1yNC#WA4tK6&!2+FG4LJ0Lrp3syrFaPSn1@Z zePc>xQUfxjdup32#{V6Q^y8O_Kzqv(;X9J`R{DO9RCN=#TXnpOlL$I=&!6Ns*vky& z_lBj)UdY!GYE~IMQEvZWNhV|1Hz2NoP<~mjI@I+TJ3Beo;Ha^}a_8u^l}A!&-XH;* zBKU_5qS@OyH#y=;m5gb!N|mFG>wp@*0nt=R#?+PW@uJlK3lnJ?cj9#aB^{1B(ne2L zB|5qRtq-U?wfhJW7}t9v(u33z5LJeuqcFpfSohh$TQqisd(M(WI$racNNcBz|`vK)*%u`}XP zeQt?c^R_UzR=!#OgF-B4jn?t?bD#c$LR_*D{j)Qbz5lOKNP8CFHaq_OVpa% zC8U-c7TVSi;GFW?)>_-=R9Bzu)|}8-!| z+zz)qmO7XY2Udu%2(gLO7Wa^Sy%lIVt>uPxRBN)<3cvz#Y9a-i#y3OdO}6_xqfe-V zPiC_EI-Uwx3It9?V|sy0i#uD+;zC=k)NM{h&w9YO>WxlJ*oxJ@ms6mR`a}f=!bc6A z!IJx&{nbBIAnN}{Ln2M)^2|Lw`Emn}N9s3$tJrg=5$a~cDKijRW2;9Rmn>gi6<0K% z8uIz-#mgBaLQ6xas!3q?=z`EfG8jE@ZI?@=>da`GCufJHdBs>m5(dIi!!OzO42$7* z-g1;6{vnB&ohzIy+zP>5ONOx(O~z}L4IzlH<}~%s46|+8swYc3q0$DcyH?CHk=P_^ zCG?l79{D4|VyP4feo~ z(W2?P$z55Z?2Hf8*~FPVPA=-miG-vfi}$mi4FaeT_GyBAgSrdsyh}dpq)=daAqMb% zN+LYonQ(~+R3UA9#hL;cM_2@#Xq!lT1IW${GjU_M`NLoriodE}tx=m_b;G!L7N|2l za;E4UV2w054@1A)tlsp&lk>U8?ow;XD(RS-fHu!dQK4Sd06zc=eyn)dk#2*fc*Pdl z=SZm$sFjHf>}y-~v7!hkH)*t`?^udIato$iydr|C@mJR=V!X~D-~d+_1qBGYsA~7t zo(t74Or*+O&&2t{@J9K+Q6LU!88TgfVZ|=;`u1K&0^!nSBK0FdZt+vn*(skIf?n`o z=7~M(4YU}_%B-{$UAtbPy_~Q@KDFb_!Mvp0qCMS$!QiCeB0NxqWN3o-?_E27sb=8B z6S0;+HiGoblIrF<_wZ85LQ{K4+y7}l%;WELD80xgX5?c!5AXI4`X?$>mK6nF39Vp!H)I}U$G?&t@(J}Z}8=mWl^$GNrSOpRGnm*&e-k<>6JZvUskkVq(rU zL;`rR|E51oX1QlL%A7`qw_%mjE3Ni#-j%QS09w8&m-Nl1M+~$J$VceaWmB}}t!;KL zaQ{300IsntFTH0o&z=Xfn7eDG(fP{9S74T0*K0 zGQ#>Jub`yc5GU}_+FrT3m}`!6ODbwqru{homVx*{i?FYS1VBZPZ^wzPXtNoy-L&s} zP8O{?oU@oyLhCZnav|YXwDC+|*(0n@-QwAAT?5-8SM5%4YFLT&e_RM~73UwNX4Dxm zT7~5c?)`e7&HMGTEHTv70xW*{~@MyUK6 z8R@g~D$B4cnfIQ3!yLI|A9 zf%yst58rZl5O`Z8mODQy>)|~uD{R$~!al}nU7Vxi*LWCIn$>lME}8R+6GY5pR3TQ= z{cVvsg8=D;@~e-LV&hY!0-Jjsnmjtj={{cTP2_aLgyQn4kRGNFjn!8K-R&#;Wd`M# z^6{FokKvH^@0ns~nRwZ;Z%R4WXoKKcaXxh@=W0&|mK|q-bWHAS#MC5&JN~6Bj1n>? zy+4Fx6Iw}ax9Cu!4d+CYrpH%)(UM*ueqjxaAKY_|%JC-M&WV^cb|RwxNBA+}T15D| z^bC=G|FoCCgD*Fq20}4soZf;4Z8y_oN`34BH$9%nd>%WzB*J^pHM?drLACy0ft!dZjOwt%+7qn5p^)VQ@5VUYFzH&<}&l>YzQeTcMHOSJtz4nORu z|KITAUX0{$G$3nbd7cMTF-^zOi+<%%gJu;hvb+DhhjMQ2 z{rluzP1sOI3FU#+nTAFqOI(=oi6o_&O>RGe;bXk{$R}Yrld=9{O$m5;X{b4LBRg9{ zjw71r1N;6sW;6(pF(Nd)GT1dSgxwg+rd+3T2%ps=+g5~P=s&nszBNOMUWh{H7UJ>i zlxG7x=~}P5M1UGs9yzbTyrlV$L+<^%RBQ>cunc^vxf}|7Xu~n9Lfin{P*cJSj+R&C zso`!N^iRvL;aihck#Ul~zRYF+NknlF(}Rtodkk?rGXHqly~N1QF;=&09U`-IY~-uJ zhhHyN0?o)P)-(g;8*ca~&VJWm!_eU0n1sYZl{YMc8w1=Eoxg^5Gkou8c>{TE9tH=3 zE@MCh2R&cm3hLOL6BbyP47htV(a)$OZ&0t71kpPeSjY?$Q5;i29An-bYlsmbLT0UP@_Ie}Nw> z>IfRq)?H(&r*A{wC-G&Ii(06})neD<#ZFJE44l@|>MiPmu=bm(T!$HX@Q>=g5HTEx zag%yQ?oZNC!Mc{RYQf~ku7q^Q^kTTz%FM;l3t5Cg>WM$pRjCx?p_(WgZB3qL#UC=q z#{(oj7(`PfmY~EQ%r*Yehwg=mJP9}P^>p`;5dtI!&<0{;$)(b~Wmv@!*JHZU;r)q$ zy!yq2PuC%;9o3g(M$g!bXR~G%=Gl}+{#t2zj;y0#L{E>7`3j-HbBsfon<3|xooL|b z!*I-6nGsfh4li7BHdMOXOsEovXV#G(rgVD z&H7rSDZq){$TU)@sZMlZulPO8DZCyHa2tMPsL#qQxw__(N}QuLhKq^~>*T?^r$H92T}gjq^87BRNR7de(ho zJ!lt}tGYv3Sak1l@-lBX_vi4-j$z$S>18v9$r>f0zBeG30o z#WiL-{|H>p5|Yg?ki)hLI1P+?^;S}|OQ)1kr;X`>cgz{->+R$Foej(k1H?I3P8Q?~ zuLmfwJ4eutr;Cc)4CgViU0Ss;T3RfCN@udbWt`84KlInFbJZ~uhR#?WJL8#)d&ML~ zk$bi)#!ytKSe#O`>nXz8+e~wx z%6kxeVw-POL*^gO>h;asN6@E$GL;Omf<)pytSiiyoSSbuO=xaMBUF>#G=HoJE zMYyKqL~2G{(spXfnmFTg69lVIIaRlc&Q5`77v?8s~o*(rZDk&i2!y79jA;6z|GHEVR&UDR2 zboe&o*En6HUW>ycdW`MPo{}oa7~+bJ+KXJun;RnF#!_I4gQsOO24v*O%+{ z5DS65TEE`D5p%Wk3q+*g!2$54UvaQbt;$&_^HVA6@|ARM7 zbU*%MIsTW?-`YveMA!Qj?!UB#MfCqTdGG$mb-413{Eub#X9g2}H{zKC`<8LE%r4%4 zqCG4lZBM!Po2U*WZBH#s|9idnBnzLe9Fo+poH=jNI61ZQ$wbRh;)|MUUuQ}#QuJS6 zMa7#G4mknNRW_)`r1KeOj+RldPBQG$Z~m6j&0o0OaIl1pd&GXR<)zr1 z#ZCia##XZneU^YYxwpjBFd;asy2G)yI~t-0qHOd^eI_W3JMTe9c~pR zWM=jUU3MzwNx`jS?5&W>02f0BbN^l694N;ilMj^IyV@FCRJSJki}}#vE>!LY=Lnu` zGxxU$)3TDw94^2?G15~$WhJMHa-S*DPh}&u#^@HK*wfG~lUqYi#sejAV zb2i~X*QfJy(FKtxC81Z0c-?RTz7TaVa zm-0%oI6l4g;cvD`8hPosxH40DYO~y-wzA#PIM?EvQG>p)p}NYkG?1P$_hU;U-PaT@ zB0xD(2U#eyu+gzT!(Pnb!Mc}_l};toQ{fs~{q52Aqp6#t*!ybjb`1QO=9;yYrpWKi zNt1gv`?3(f@mn!Sv`pAH<6PUyY;M`iUZy{;k%N9mZH9_vM0(#SxH0(BI2%UZV^1v) z+I2|&>jkFqbLIN-_$*NC&+q$vbAY1fET&sIExjR0{b`>kTP$&mmF~<1-ibbR)|k9I zuCEbRK>8slsX!JChKZ$34uRH{qv`Q0W+-drod~0!4&Av2Ig7U5IkBn}1FBbpmi=A$ zUSHA&cOdjx?LPh3&xG!$;OSBTIQQsN@{ndFlP1?P@K*Jx#J>O zxc+(h@0a6fa*X)+2QKkG1L6-0V#F6U_Q8?8rwGddOa_WB?Y$@KG^`!l^ zkg_3st-#f3ih_?dcM_Mr2BK()V;|8fjx%{{x|3n2N(c5%KTH|uW>CK-nm zjT*_!&BiUh(r)TH)$FJ~vlcR)`j8~pw;Kt`4gmRpwBY$xe6-fmA1j#=o>gD;*!QkM zgw+QlxH`p5VIWVumwG|euUX>M{>&}bPwW$ zu#B6SDqo&#s}EQGl>PFaTBBFL5!w81gnC_xnXL3PCBuVM_mycwCB`S$ky3)HQ_eHM zAxNZsL*RmF>p7P)n6nz|wMi}l-g`Z$`lamIa(jg1plktskTTFM>EPumCrv$6r=*KK zl8sYr>Jg8v84j&5oUc zKty%)kjbDn#*kAirdZRAsxOF#8Ai)RfN;OM&MWLK z#AA+>wNN^1l&Jzv2qna`IPl(iv>>)tlHmI6we$$`zo09Q(03fbDxjLN^yppi(DU*Kn|N2 zb_J;vN-ic#e=~q90&-2m1~Rh(hl%kG%hCcB2}2*{k9t^nhiua?D%Bzo*_t`|4J>?K zR+#ngoWQc=L%SUD=QO7eX&!Gs_7WZ<0r>|1+;!UJ5$_>a@b0i^ISC1q38NloFwr5^ zy>1!#&FcuM*ZFA>lAeCSiTR98275k@{lDOi0brqcEcO6K$97E3>evLrjB(8-j)#>q zSIB^=mm;Aygskn-_P)1{V&E|tSP_s4(b=7jZp7sH6;MOnp5A-VU8c?Sawb&X$4aGl z3v)5_11!{II+64lS#}^nnO4@p__0Awlu6 zK{=7SprPT|M{uiiz4?S5o}y-vs+Ymv2l2&wN>F(Ia{$M*Sh&Q$w$GgQ zCVOT^8P#}M*vrecK~?*C^U?IqZOOwv8``&vEbl^XMtLP1$HQ#kG^<`IU)K?%ZRfavh_x z=*Gz2?&rL?(z7773ZidQ4D8J^#4&I^Dmm!yAFet9&ZK?s+0<0)3f1HUO)O?klHD`j zw~oO3^`7$g2$P2Dk+Z!I_sreqKgG`^az(Du#0#w2;c$ygNSm?&adkk2Wz8Uk>@s@{ z=7v!G2apI75YHb4Bd)9U9Ea!fV9uw+wmGAKT_zvLG3T*m;}1%1@+N1b={D8u`7riW90-J&5bNQd%!M4pLGrn zY_^R#Ny7F4*Z+%bcZd~+0k$oVZQHhO`+Uc?ZQHhO+qP}nw%xh^d!2M{Pin9$dsvfN zSsIEE&etG14$;8$to!Q{LCOW$bZTPjA{bP3glFH=9Jjlm?E~8R+%%3e5B7~fchsg7 zw4l%4|3J#JB9zrF3bNC!OKiQv=2MNY8=vYo^8b@K$lHsI(sF9c{*Q~Xb2duHy^GOx z-H*|A-5k>}CHDq1_TMIl@jq+hIp)9CSnU5XK)x~>rsVMF_pBez8>VJ&G5*s!Tu0YU z%FMoBhqD@{3wsD^<*mr?4>v?x*dSo1&#y&Xu^ z@+yCuulR7TP)m{>GkCxoTzwlUy5&S<5g`NE(AmAd3WgG`CpT6Z&$l-EP{h;cT$e5| zZnh3u#-Ur6M28K%#sO=Q#=&UBGH(UY15+mJqc2X*%>kDL?cVS0DB^L`EV;;(@3=x?Vq16P76*g~#jq5-tiEU53>`eP zEf>+af@K%o@|cL|rV@q_Wr_)ki-x%@%uG$s`=82B#wO}i*9jU>$?kthFRA$DRk`Zm=PGG6UzR5B8aly8Vjym`UK_a> z94Wgh*C3GLgX5Y-g~pJoEhvx|R?Y#b2offs*Yvr6YkKqEMIBb; zBD5@2wf3quhGOfAd5G~?<@h@kxHe<}HT<9K`y*3Z9!ky}k`tf9+-;JhX-lcQ^6+$M z+ku4-Er&Wa1kHzTrMp_tp&`s5(7MSP4Oo2(>9E|YQjJ+dscQ%PiGdD4ag8u(OSva<+EwTD{448yyJREN8g}vMXw&T{qMzdh*mq(pN1m1(P<| zmX5Ks$QLC;GondenpH(Gh`f45Kp%TIWP@k{FRi`azax4xtb*G4eSBf=${yAbNb|}w z3{Kf5np03%8mM7GcEZ=-M=g=U2j&Tnr>A+Ew%4VRQi;|M)=VT+vNr(f^8qEc@rjLE z*|!7VCNoV;$1vG9uCAZ*X^ zHrYo(T3A%aJvYk@lo1FkmW#i+ISkCB(bP?C8oj<94LcQ9&6c13*)fdaY5vQE1|Sta zx_yo=dnIjeE}gcFibq2a!Ul^zg>e)7HQXhcshIvI@*L-@d@|CJV``y3O|AnJz zuV;|ZLXl~V_S52fqbPBIES-CKL?k<@b(tJIAIET6H>(ho0z z=}GT(|Fw!&O`k!t=&6>d>R|WENUJ*G*0;TbIrJ>?te}2{>|2x~sL*l&ef>cYA);0B z>?jkrL2V{Z#g@BHIr&a#+tYRmntnNXE4S4jc>O^W+Nb~6V0Jdo49xm*mN7k}Z+m~GXklF`zXOsX)R(=$|gZ2CYAnR~lBEE?wFh5mve)&fPv5d>Yj=mP=C=F#lc}x;!>*wIuWzhrt^!OctTs7d$8xLMvj`yCx^~YUMiVx< zbgpdJBO!ru@iBnX43-q{lC`cSax6oHzD)1i<8!+>GqH<}@+W~i-z2Y1-`VF}S3Sw5 z2c3*-dMiHF$6o8XAJ!u1-B=6c0N!ht_^7p(t-^#C56|Z^3uF|6A_9FK>mSwS<;(7} zY;6#zN{OIxHoDWcW#*s-RsMX;1nkq+px!iG@z>Wa*VpkzH2q$>&AHx~R&VncsCdzz zg(bk1pGVar%@}lNHbmk>tTpwPv7=jOa0)G~0NMEl9c}jc2mPZ?Czs$KwH6Shm%93B zn%&kF(cSe>e8`t)Lt z$YKjG`-*t>&x5dn&0d)QqES$>4F)WNhB>I^66EQyYmk-cJT`cUPa9|@xGaJSlq$DF zCC9*>{2q~DB#d3WnsuqA*aq3s@<=e1PVoWx_mq?Kx#3$61G~r<1@VVkM8U|gUalJ8 z*ZKWOo`Rf4MY#3C#V4?XbXIl?-#uHfzh-*?Ikb z#a{S&@4Q$cE<`YCVz%%KyOJx&sdw>6Kb1%vVKyo?dT|-*9$LK1IGQJ;&&T4ZaCds| z_n_^roIXCme#>Aa&E$1vx!Vk|74pJ&c&~0m6KLG0eaJ(yb`nf-fsTkn$*w)yRZH6| z6s0Y5z1lX>ktFGaqqvmnytjIXP#D9wf@Yks2eoK* zsZ>0%Yagn~$RhqXB=^IF<3?9g8OsJQUEp|q1j4I0>UMsq{4uq0Ljb-K|ZMx?JU=-WDu^nEX?LljE zs_^8;FPDN>jAHA|Fr2lX_N5p=Q;g+2v1=WPYuIjDV>GN9Ez=|2ai|%${ZW4pHg^r- z+&*}z?||q5l=z19=W2YXs-Qtj3bundU9Vq=Gw^94o%Ribe?I$mvKjbMr)Wr*3dGIe zFUwSZ8lkTul_@w4tF0m!UZulvIt#*rC#xG?giJhY>32Yr*ua&>WqkOzo`Q zdhj$IJ7N+mPdzqrpLIBldZ0?wP8oBw9v@K)-c8M_ z5=2*jMbwaT3PoanNSTwk=V1wIKAyLeW7!xBzh%Pu}jh zY&7SY#ScgHg(%(!P9p!v$D@QQ8`k`UnojH`)jrdnFc`2K&Fd`a1|8;e(XEs41hLx2 zBIJ$&6RZvB?yVpYhv(8%%8JHaM~g{TyXK>T*e$Em&6uCM@9OQ&_MvG4RX`F(zEN9{ z?+j6vf5w%p%H$kFw(D|R=8m1DjYzkV%v``ql(sGz=1(eeZeXl2basIk4IK;3%>R*?ue4+Xa#Z5TTl4wEM0d5+9Ys z+HZ(%VM&9Q6O$jEr<7inHi!1v?)@j3oX#eX7H@yJJApNjyJ}l8-5{^pU$vd})3LUq z9i^au5f)_s8d@k;_~ng>T0Q0a`yDK48FviIW2Bqs(GVe|*LX0SNoxFz!Kn4F{FsR5 z=q>3T=;1cS+@=-ikr!tR>Y#t?HlYeZ)$3_x*8{WJQ)YAC`a<$%Vsn=DJ@lq*G`h1(5`XP{#AV5PJQp05G}#!m}}4x zh;=fadt}lm^e<`6ifD=LX=P--?_i(6xLi|(Be2QzfL29+3Lk=4r4g0J7x8oPQ6o%L zX+7U(n9^WCU^u8O;l>o#9+5+~?`6I;qwYwvD7s z;D$hQS2jn&A2Xq}a=gW7dNf_@7OW!Ip5x;)fTS@;dMg5MU1D8$!~(tjDIH_NE%+tU zo{-+F%PhD(Tlp3EXt^F$>d-}OhG=D4Jj9>{oAk?Q8-7{FY0Zw~VsN2e*cd*=*{ZhQ=ws$F`T{whbB5MS#KX)NUUcfF#Hf6naThRU$k^w>AVX8O0V zH}}BBX?$G@RKdQ0^W#t(>u5S~)au5Pb{I=?wp+lskHY{RGxh_*PCHUVO{DQ0t;}Ln z<)^2syql+^r#{D4`yC>9U1hVkLw^kjN8!#ijeM3BymT9ca&FMIkyjuY6BPYHS}SV3 zQKGlm)H^e3zfhO`p_DL^v0bEv)9tQDSm)emhxwBb&FM&hVU?rBNlUlW*8~yQVe7v) zw6;s`q%X((&w%}rxdh8tj%hf``k%mE#syfZtuT<1zSC|9g+PSFrspTd-nNgE1euM| z6~SW#;whbELji1?(Lyd0!GHz7Q%_4lb!a7^BEJ6h@@Gp48N(W{eV)PRy*2}Vb2^??&sHTyt-jMOqYW{1VsJry}yZ+60Z0X=YaVvj`P|O(n%K5$>WBWH;K`yK|${>Fs zC>u@(eJFlNzj~?Q%n-xev3hCXz!1aSBYMf8e}sW&l~FE?7kwzclymh0;mi;N>-*Rl z1LLyTSVn5}mzffC)=ZIUyx2GzckjQ*hze8F+p9>)br(ydK7P^wL5a4bMe<9~w+vZQ zViHQBqfp!W9WZBcZP~``2Tf4my8&S(E}kF zBlu`lBz9`T9)KXPByi&Jy&cHIc*WQlpaZ_rH)cPU(YkI@5VyPR48|9CAm|_K z#(v4k$-E|~p$*L0=eCXe=<{v4(dHgI$g#dj2=8$ z)h^e^<3CrUQQ)Q+ykyre@4u2j$9>;0)*uhLUGy7Z+lX@In7Myc+ldl$4&Oy8~4ydmMD(lp3RDU0AvC@`W|UMR?B9YXy# z>Pcpuelzh~fQRWNLprHd7%?@EkwA^oBFqFJKgbaifIKBn<(2}@)+FKdZs9kYX3Ea* z({Nls0YT>e>{|&@J9Tu_csf8dEB9$FP0ENX(vLp!IbS}`kFTbWyx)hgrJW1C%)Gnm zJf*9%m=`D8ADWxJsinT`-?j&}IbmXhQoE5)>ReD!)vq16ekke;G>jWX`$)3ix;D7L zaTlLMnd*Vhs8;JGh~aGO`e`hwY%NV2(_q?FXh)bMbz?&x9^;|gjRe3lSX-b~B8yQ- zv(4te%(Po9%x0oBc{_^_7UZikv{4caN(zk?jnb>L@lu|8%&UGk`Jbb`2u(FL{|7~2 z5j*=$m`ArLC^$u!x@ZpriAfrd5Q$(hIvu9|zybH%*o$sZm^50|fZ9@q;mwfpYE3%z z*<@1I;e+I8sd7D!)BqyI2d&f_Q>+X&hY|{_%GwfLsbt-xMJCcS`M#AF)|XFRVOcJX zlET_rv&$P9mLQVPwbL2!?>l%?Q57(H{ zFj|itEa77_cx+z*T|i-Efr(92;N01~LpyNIpRJV@Py^9U)NYLUKkfRa%ccN5J^her zX$l!$TNfvV2KeYuhO^LT-fxUq2RGhPkwX{Hk9in!0s>b9pD>HI{sc?S^dJYZzkHjkd-L~_gMHD)3yN7`i+J#BdHwRTiRc||4x#(cW2{eG?jc;&4ZRw4XP zteC5rqV{HZsm4((F!6F+7|SY`7KJ~#Y&XAYWyy8qCiWHTXZRM0M1~nDb8#3PH61#t zd78phi)WXeSy*WGR*uMM_e^%%Q+ zIii13QCs~y&F`We}-%J@@#v|2{YCngD{U&~pc@G3*+ zMjUSXGS|RG5VGuu&s<`6W)db`#8cfxZo5hpcOw?P#JfbkHoYBfDhj`%)tgoj7JykL z*g1wBe3o4dJCLV*)<0!w);8A24Le$lP0p8J&tpE07tUiW>_SbSv&z^F z1mQs@sU3!hU|HmS^W)t=kaQR^DgyLtM;uoA<1MM{w8Zu(Ko3JvvAR=j+oDtdA(KpC z(4L`o)u9~&)ke3XzZ&|sQVSI{og+DB7@(dT>?$dt({ zBw|4x!MZ_=_KalPIuoOR3H(k$^c0uXlvwKg1W&dEa=*dQVyz>YQY9(K%%ocNQ+up5 z4C(SL{?cz%AzhjMu$0)ex-`JxgDm&rs`G4reC_ZFj(_eg(vrRlL}la#%)!BzcKrQR zaIPG7=8$s6!zra~E^RlLXl96`Lqk(6HZ&7HY##3Uz>sZNfE5NNGHOOfsyMiTy=q!| z90$zQRW4%OB*5%CZO|l)Sb*6gj+-=CZjJ`RtRqN7lp=~yMv)9|lsH5cbF5E@1h_0O z3_3!2=_R!cU-q^wK4y%TJN`F>oH4$17tM2nSzCC@x=`AnlMmpENn#i@Na%ZtiXc3g zsYWIO#JI6ZuJvVX^+#`HMIM9t5SY50_sBClFM8P4BR<>93gdbP;5OgK?TvgimUIxa#$X{lPH5z}GHA$Z_ zE@!X3m$G}crG@%^Jhav0^Lsm_)nY}hN}845S?2>63IDE2d^Vz`T3r=G?J$vB=_WcT z`88^c>^?A+aW)f)XZg9Edgyz$kuYoPT~$Hf;a|ts&CAo0xbO@tt$vVf^jQ)f;8-bD%)*RyzL!TtdbuWW!3fn-^aL#VwO-^7+6&jneHES?Uc$^=>cHAhnKS;m!6_R9=e{rZGm zB}6-26L$*?+QC%i3NAr5TdFJAc4}=eXw;m;EToEx2C;lVj8-LXcAxv zO1jdH>@wsEuj3jl`x~wL6M7>ydU)+LSnx3dv4`lCUT9Nlo)uQJhhlSdeT6K#l|MBF z9w_s9^^+6YA>4;a^CmW4IT7N=6N#Sffr}|_UoylFBhly@*lMdtK)9=b0}-nYFD8W( z?in4sjW4Ae($B|B;pukyBp$L51PE&XOcj6&U+VzQGS@f-=y?oM(DtV--P`-~=(#$w z$o!8l9b3$=Qrb0DuHG}rm`RchUbjz@$>EiC;Bupiap2O&Qk{2Zo#=z3`+IY&!|2)b zcw6t6G1cj=#Hvf9BP)nF$xq+3t^8=IbHySMl(=r$4I0-k>fhbieOi=f#)C1!N4L-Y ziW;`=yiq#T@1fNQY-Bd-O2eHttgvF+Tz2zJn4T3kY+qE8*E#AMslr@ z*C5O7q+4Q&6BVgm*t@@Nl>N>+(dqc$!Bb+pIk^K!e*f-wo#qru+>xVTFYeeeP{0F8 z>e9gH2(p_*%lS1EOFk-jvLE0jKS>tGwnvn3eK_n}tH%!a$eMw27X4orczKZb<fAN$=2qZ6P*|(^Hku6GUV)k8hiBfz$yUyRpJW<6Au9Db;IyRQDiirP zD5u|#UMhD>g}$F3+aGeypEKJZ|I-Nn@BZW-(;PkRmMAITVU+0;J{jdNSiljsmlJ+S z^@942;)BsfA$k1A52YN~`2@g?#?S-+>>LQwohO+m^Fr2vSt0vuFIoHtq_J+AHx5Ah z8zD9hpI%Qdp6N?Dz*BYpRP@iyir#mE5!Ss&Ht$HGxyw$bnkeYB%* zdEApIhV7Pcv7_ojvZ^zP$avQ*xz9Z5M5J<#bpsI+g4?z&`;HuZix}|@4~UgW3?W~) z&X*Gmw1~)ytF!hYCKQJ0t64sWIbnB1s6+KX*|Y9)b&spF+n}Y*aPG#yB5mc3Syt}9 z<S4;RLeapDmg9*lc@i*d84cX;3@m0pN{IgrQrcyT#3dMqCfgWTl%etXts)h`p` z@7zy_avE2!%qBHmVB8@LGRGEB2L5O4Vp(MSepeaa_<8ghS>O15zTnLR+f05>fL8;M+Yk!G=|7soOZxtfbwVfyUv zFtmX$*x~O%wGc9+I{kTYNw3ySCL&1=alpt&q88YTKZa$Z(7O*=uC@^#S)tYGl@#ci zQJf&UOitkX{N8sCN7j9q^jN-zqvovh^euX6`Q{^gPB`P(p}hThE85)WF#A-NQxfU* zz9kd3q2|++U}_(zR_kXmGmr8WYc|%gx19qg4yMp)`w0&aPxd6Jcpr zVP~;JcX^f<@KD0}Bopwov-TWQ@I-0$wwt`O6x_b$*UG-YgkpWsV;V>dRkfL8#;BqiJSGu^J( zE~6h6RgJS2f~&1mzr6KS&{;`Q4VK$ zO{gNa7Zo@0;jXU_#^L7hT7|rQzD%|gwJX)IXN1-}PdDJx#uBw3T>!^|0qRR*UtMGW zrXU4WLy1yT?*#G|UT^99dGt?lmWmc0WJ`Jwe zGpciZe29`>T2ZPL2!`H$jr#IAr31h&LPHOr@G9ezt#1Z+x3Q1G-FHgk*8vd5O~j(I zGJ5T;@QoDVgQu(9l!7BJ1>Q-qC!+{7qO%Id*EKi|Bj11NyWu66#_p@6pG5QYEYQ!3 z8mp(+>-f4m8adX6Tgh5k8)|-|eV?qYs2;dd+k<^Fg?vjaRfU16EfU2zOllr+5$|Ute{^x>A zk?NEp)&k6^9=JTd49i|O9EK;h-n<@PWl41jKbI#FxHwW^3}yK1qv-()q$5T_N6;6#MS4dGpB3caJNXk?i4@S6#r}@y1Kf7*HQy^2W zvDi*NYuc@TVSOZpI}Ahckg0z_gAw%l=tf>=`Ybaq;Ccnf6@zs>aD7dLPsq@HTiM9{iX5Nl-qE(R zvqL+*t8~54yg->O(B&CxX)a17 z6QkWNdK(=^HlQ~euizSi7RN3|qOy3C&>a8)mB3a@U3*u9iOXD65`ww!lrZA%w~YOB zcAYapZ5=WM_D`FVuCEqQP~6Z+V|%NcIe?;5ZMu#3icBl#Y}+ZFzJO-g$d&!O6z{Kl zRz+#u?+JxwLqnBT7p|;y9?V~iVP`7iCPFz`X{B;>{kZ(jHm)h)PajLNs}o(|j4P$JmJbo9E}r`nwqO>?);dYw_#CgUqDCo@ zURq%J*QEaK2*TExn`PDK^IE`J0S@TWozvN+#65xhd8(V^l*x+j-gkAI!GtDt<;c7f z+IvEVPi6voDgnRavBMh=0)l4J7~O1PdDz5Qlwl$)4oGl9ZPaG=(Yl%8 zBmb&|EV?`%gjq*FP+)P_RMsHCE-6cud*Na#c>anY`G*w+aow$A%gp|^`|Wr zMo94fx&71YX*-nPo^}tw){d)a00Wgg+|{s4vywM!E29+|6nHuanKtaoX`g~x*>WJ@ z06@fb-D2GP;JkrNG|0O&POzndtIB8VdM%(K?Ib*NS5W6Qhq#cm@l!@n9aWMFtxidB zLN_TIo|>^O>7!GHa!C{z2BIprD0N*37bTjXvs>_kr&ucZaN{)XYl4qfvvFK}IS#g> zofsU%zI{--3DAHfsFhReULY3UmR^pYL(VOjOPy^yp&3ZJ61hx5K5x}Nc%U7##M+@v z=|GC=V;O~XEbhv->$43zxeM3qAvy5a^YNvyZcJbsiD+GRIpva16hVVLrdc%g>2ZGV z9=kw2+1}dy@bYqhj-O2*WB79N@^ZhwGhKCcbi3a{dB5Lpg|l||di;6cIXxfZ#>8c1 zeh!ewIVY#0tmr2=X7`E8CBSiYS4lom-k^RR#yZY4*g}+ht0^Nn+FA)FhjKdg3A#JA zV?<@3I!zu+8{MSw@?k%90n+;F7tX7`Gfl4p5G*}QrHs9=a#=~5n@EX;S@FL+&9kJ4 z9g3gM4u`2SUCwN}m~p7->MuC}Tj;BZW^Yp8$11H&Ow){(`?1_Lt115yA?DgJ&y($2 z&&0@ptfmWbT>@Ji*L?=nE5r5;sq>aj9={umQK>_7LUars?G9rFNI5CUA=|AMJv5sZ_m4K1d>4D-*S4C?G!&hH)@%j&) z)H1n`PVMY}cB1be|M7icj{O|pMfqj_iYg@}+4=guJ+1NU{d}LEMJfHJE%kl790!Z& zG(;uVy!#XS^(cwf;m}u(! z-41PCVdCIL*7poqdSHtI0Q);giF~YOgJUnmrevBP%Wee;uhhAx0-EF`nA9wk5%`1t znO%hGe-by>h_DR<*0gBF*8$kU9>V#HttJ|vj#-I{CsGY&+=TZUO0iTGv2*i}{?83W@N=%Xp6y2g7TT5^XTZKm~^ykJz2|<#U=L+#dJBmee zO?leEvsm_mJFx&rsM>JnZ^o?D6^yXQ6mW?x-TDQxnInrXWi^ z-RX5@t3Py}WteerT%XcMHgLI+&BPbKs!!KjhW3VmDnQG~5+iUaU&ua=q*lbW96bSA z+7e_pHQ}(+YQok+lPK80BxY+s-S>hOqV63@+{7^43lY2~%pTr5Vh5-1hbs?Ip|peU@~L0(z(5EQXk6~IN< zfn>9DM1)e9R?vQ6i}rE*-?VjH?18=?bbvWAv6&8WOI4M$W}PekIl0MkCb>Gn+PmqK zbc!q(mdRF;*)H{G3B%f-@V$G`XnA8l^5%)?8tcMCuI=*pk6$$R0O5yxLs1tfPUSEe zS$gkRNczk5s8XlI^{48zjb(^x;Ifu+xfmxVas{s~JE1EkUMB3%3VDA4b_;igi+$Ke znvaV!1Uni19siJJ06ONei>toAFRtL-X9!4)U= zr)}A&1_`on47$41B2PCYd9sfgR-9xXFu9U^PJBt;Vq_oI*Y9K>)`@d?Dty?;yo)ub zcG?H270NAcGU^SdBLJ~=c_Tu#N;3B0yoIgLX-*umu=7DA_|T53*AhLltRVkd3>ug0 zRUTikWb#Ra9#w9EA%u}CsOO1jjH*RGTqKcQ2c+nDI%0bOd~c4VW>(VyjMSBI3y7!> z;<9;lBOWm-N9toSxyx{LI4JR4?7ajHvD967X%)F=(XzrW!K}h#fDqyA{xoMYtPK`( z%?BZ%`jAfATtwHdrHreKMGqseFS=Ol?Jptv+w>P~;gbASU1Tv@Qhn)+XvxwqS3EmD zw3Jc8f}xx+U&ix)9L0I($};d#(wA5_>UI8%%Q+=e2+S_QpC3Jdlmd~tlKTy~ii+42 z+24D8nkxD`Oipl|C>eQz!w2bpgEdltk8Bz>Q@-I2Ql!CL7+Vnz!9D0O+Gn7%fOZT+ z-w=@98{Qw|qRi~sp4ul|Go}q!k*mhcJ*?4NbOn-E7m9m z(3V@9zG@DOQwTaJ+7q|KB-{-WcVvOnd#~v6R?xz+%$aJ9la;t&6t*B8(1do{CwJxHC(oG#k1iNh>OQAZqB7Nz6eoV5kaxU*vnIqD}mkwCA!!5#uj zH=|jedU9Xqou7zvj*$Yi`C$9QAc@AY@qK zY=#!RcqT*Pm;q298eb;VrdaIYZvs-%I8V&r_>{i;yf{nW{oG66 z{mz(2{*owv{jT3j-~T9o{}vAaYyjyLZMi)kN1+b97ZcXNmRJ?W@zRg|?b*5Xoboq% z;%dBQ{dZX3d8I1$A-%#5s!czG#=G7JRExdnv-ndCHza!5@aE^C^x&WKV}mkRt^2R6 zO0?}IFM)#Ulz1$4o#clA35+vP8^QX8XoK24Ia4*6am^9qdq@pO?#&d|Xj5{jlZ{=bl*lt_dx)>{-d(2@-?|Zl#8^FTP^&iE20H24#e(Q~3g9nF_W^r$%T6B8W6ey6`i-<^Cw8ke6&sGnN~XJb?t|hlIM* zX%IbTqh90>*8cFiYil0vbf~Qbqc<8lQ?MH@ABz7LQRZr|*q|m${X5dsR+;lo{&@4j82FhqQ&|q_soI9*UtjIbw?%IlK5Z0HG7C4RY zD^6!`v;DgVp2-!E&DWM_SH>)AT*ObN;boQ5(2-D`xrVc>P~Y8mSS=gw-!^1T0Sor) zTRg}^n*$RuBM-RqFHe`(WF z?H+2YUH5shez1PNjm?TW%)jNi4R*=xp0v4tbyxgX8BD= zH@FVa&2r%WF@vLHp7kru+FS~K;L~}?j^Ij;++L+mpg|O+$|AY$EO%esmILmDir5T- z^5G@$^mQGEjs_@7 zU4l-S1oaS3ig{S5QV_z${rNBvrEaZkl2+Bg;xG6&AZ`4TuckIbLb}$@_r!noSGD&$ z#_H(6>^D-$E$lm4IZQWQwf1gNmJmSqBD9IB$gz1f2#I&vF?`L)wF zA!b}bqLnbq>Y9sjAX#&a_~|?!TwSu0Jsr$TNmacT3QJb@j9s^C0W< zwL(gR?Ga!nsaoS9m8x&(++30TofNnu=&Y_}H<9hmE3+id{BLKWqG*A>{*RX0DK>>7 zO?z+5jvuEC!boR#W~%GMo9`~1jxJ1#Q{zB0o6nyxDM%(yShnD`NWpGee_H)ns%nCwL~Hv zoBW8o1TRI(e_9J{l$BL8-pyIuoGc`RoBs3Lr9gE{SldG|wQ%#g!K5YTD|(UXx?_{Y zi;Dq`f`>LmGreBxdt*CaCB^w;wm01Ly23pXo*cfVrHDnLr<{ zFg(su@h!Ba9cU7sDAdLD}D~1#{KytZstYv2fG{ zO59kK6XoBFV5y7Qgj(r==#Xjz#YCiO;tUERFxCp#KDQ>TOdll+t`9a($}?Oq>`jMT z8%`%IUsMivFS^{mTOy}6gIDuKmw1tE9qk9!c7IM+oEe=NYDhGDvBoK=nM&b1uJcVw zSd8fHzCc}V60YDJA0qLW$g@miTX*?gUkEVrkpx)CE39|$NY5R04O_l$l4Z{*-M@=e zsA^)SD_pT5kb_;N>c$BOQc_yQ1n$i1IGG!G<>Z!EGs8l7qe264c8b>o_fWe5h<=kM z7jskrH8**^SI(pBrY8mou>2E8NutLaQA*X>>V96ND*i^E4l3M@?%u) zo65-@pu;U&H6R_xCo@2gz>I(LvQ**Ioh2)#rA0=zu-=Rvf)0P`o9|tx+2;qISgaS_eI6i8wv1R^G*X9)C9}ewUS?0C04!To&@IuhI$i#5f*K2=md0s zQZX*I&ka9l(~~>^!yEX}K`%D&g|(?ExH4>vLf0BcSwLOg%_eFB#V4fgO@g?J)Yqzy zor@IXtInW@z-b(BzyLfS7g3#{04+dfXg4u&T(&83f8Y{b{8 zsKH%idN%8DMO`6^V2RaoGxP1iO)pWE0lVp~^|2VMNWufgj^s&`s1mC$V24@zpv~ID zG29%iHU`%rep-T^9SD>{t5w7s!P@Dna-T>^1!;78hytqYrnS+lqKy$~1w5BKN7kqP znrwW?QnurIe7Bh z+g3pC`W$6^ibvNR{QKgeLnJZ6KMvNMV*ZM0$iW0zh@+U>(dMB{iF*j=jlJ#=66*-< zcFmxg`_@j)6{Jd^R|S*HTxriL6=13knL4%_NR(WDKevz7Uxf1|u+$3y`M+}Jl|Ko5 z8yU3hLfi4kz-W!9E2X-0NC{f)giaG{j}?4HPOO&bAEX2-Caw;dP=${yU!eh{GvfxF zE)cVeQN{&*H*#3xsMae%&Z$2(_eGi@t%p5)SozfI1hkj3b0}X`fn-78vNF`$QZkX* zGkcr#5|!@%;It;xUSHNSL(J%;SGh)GRkB3bEy1}GnrMWH{0J9~pru#Rf&f0hLw6h5 z&}tCBxQfI#?E^pTZ3|)q5{hKD30s0wDJMR9urhXXO_~B82@2D-{9bRYo(?Vov)e95 zWNIpRg1DGWV3Dc!;qNqD$S1SrE!SJ^OB$11?w8*W%&^c`!mR*mseU2~heFw5J8OBE z9jln?ybcU>&{Ox!kKJrxJx~k-XE)CnX3@rBxPrTUh^Wi!Ug1rzb4er0qb%QtpMjvh z@L>?AZnc1kSJ70iyfpyJ5U4NHLx_1+XSsb@wsEXij47sUJzx5Z10mN0ud@ThX5QFb zXM;12@Gg`4xk&48`IUgGo-&ZsCzAWQT)mZmu9Evd-65-k>KO}QW~OG<@bYn_ZL!mj z2Lw17g|^i5ao}lY1?a;7PMi%Kz_55BD9WItm`SkWKqoH-4q;e0(J00MqnPI(G4itk z)w2&8VgV;Nf<`bG^GAVBh6fH|V10omF)=QS&15`Bf0-{ZXU!Iw$BWIPargd z0uC2;n%CE2;$M-1OOA}Z&n0GRi!T*ejYTvq!4Z}HQkD5x6d6^jZzc?upgk6lz9 zV3gEvqXz+D^{_efkx*wO0Ds_Gx(3e)b4=i%6tORekuwDW=^fP45!pL-NrU!aXnATV zqSYf-Iv*o8Su{zcS`Sr(SDZALS*Bb;itj)<>6ZnL87%B!0J@BD0_|9R$-9f_py@4> zmgMay=guAD1ppG|k6>*^I|OES0ah3q#<0W!liaY^;zUS8t(7g5p8>LkG5Nh&77xH>t;Vp0C_=TKT;?~P8@%U7r?(_4lfK_<+)by(NyXkc~ zZujifbE`!p*97%2dsl|T2k5T$Zi295T#ky>}USliux=I3n^AG%E>V zDT@Up5dcEhSgSivXikPO zl!UFK?6xxJaL;YJXoV&5&3(9!(`1EuNW{gt^a#wF?s5Dx7Gdh94k-Jn0s_QHSeU^_n*ha*<_+$p43^%y01X0IXnv82GcMvy z7t6eYOb7#@oGMpvT3a|t)CipRp1NmQTaZJ1$CgTaF=zH~Wu(Bu@4Nj^8Rmxo$be9p zu8zBCVq_o;{KSq^UEmUVWARSm7Tk6?{^cF6@1wrdY7a1f&9<~=&@d?0e_ew3>+NT% z&s*XD-9L4Q$Z~(Q`{UFIQTfb;)++==Gw?f3ceeQL|%X@Ly_7G^8v8Ef6Mn9VdF@u&YPG{ zBKpoisXi97bBkY5i=~hVtu`Bi;}4mxoWa)V91)y)`EezGz(GJYP`dD|1Fl(mDX;=< zA<-^!&WH?*kG$1=t4iE^_#fqq>MvTP0KH;O04(klB%P3h(daQr&@>K=0?1651EBBq zWg_rOWJOuM0P)ATwdkE$k5(3nmqD=#j^L%9Un;#NskJx>Tp%JF#S;t2XXQxd zGsT@flO(JpKji=iv*c5jx~SQwtRPiafmTF6f0n?eX~3Q6P*&`k^Y^p}S7^#c5_U?< zkg!#+G!Pn4TYt8H_@VyWSgS-N69#BcQ^Yw1qmZX1NbA?9j^g>6HE%L)I#su$FwM48 zGDo2UqJ2hvDga{7BFWUC23r3H)~SSk)V=7@Xcb1vA93}Ie@RRTB$biUyv^1ReTRjpsKzf)cQrsQ3GQ+tw!8!Cs%V1RS(rbV(;&T4XPjew z*g&(8i>jmayPofVVyuStp~RJejxLNUy%{-s>j)Z$&c#B{zadN)O7M{tE2{5X-X5wQ ze1$Rdylf3*T$<&r3p~Qzg$}fcxw))rfjKoFQlw2w8d`B`GHZfcqY+LAr>Hkh3SBr^ zaZLZTV=F-uLK8(z-ib8)YF`?*K5e}c2stZ3tHc+AtrC($8x~HbriB}Nd$^3bZI;qV zlK+MSnnQQ<=b@^}QQkj%q6;jOC{Lhy6a_+2aO~c+U$#kjwnU2;EY!bx(5n8DJn4ui zT1+kIR&JEw0W^S|Nd1e2-h{=?e}^6FzV1tVGtZMfiRfhTZk)!ryzyz|akRu!)xJ`3 zDXR3P$TC*KLg@_Zn2q1qZY8kiV+_!&^+6crpNI1sSN~d;M2DAfvwO1+WFO9>k(}>{oA7i zR@!NL(Ij)k&AkKVY635;j1Vp@`*z$(>W!#%bJ?qCz2mU=+fO_rAYi-W9>CZB(+$JA zI7hG!ZIVRn3d6ZgWL#9cMB@`TA~r6s-9r~gH?r04&H08GY2$=!@H9%M`oYnMBn$I= z)y4kV@G_lmDP-I4woK0Kkw-RZ46|`V7{z@EU;`bn!?0k+;H-TTO~fPYSoGw~zJ7|? zg^(KBXCJenmqGX0%`Oxn)IM|A<%rijP!0V-1_gb5#zcbjpAP*yT;@^4)7*62QZ1k; zK|Nwyv|&z=t=QI?!4qz9_p1#IbcPAtO>|s5bB8hwAS3H=Hm$(}6=_FGVpx0AeM`8= zNdXliN;7Y8usM{Ah%T^^MtZPK0+tKc2rfJX?AEMu10^r(Bw?&EwHfNtCD@V2uMPT_ z!US?;S!=e^TPi>XOm_thpGwjd9{2I^br2$gR@Fsbx+{N66>Q1!iK3EY#tV}}CtS59wx2lM>+G2~&&eV;)V_ByD z|1fGR7en*r>q3+aGT@CNFCvT@pOdr{+pRXm=coO7L(oZit23>15=7*Nf$t7sNDIQs zQk2LV`@@VAfGw@czx5dB5Wl3uTW$Ud1~|2+EjU2Ts{WN#^pt!jabX+2(0E!M+IB?K zwmz=C7+T?8kdw1)yRrJUG|!lpVTfYYTaV6|UzI}229n`%hd=49lioV%t=;FpvfgS* zHV00zk&3!THfyp6UQsL59pqR#ijWet1zK7V1$uW!cZ&jXr+Be8R`3s|*RMSMS ztq+3xRY&XTTdyZkjUF-L8A8cCYJGhywQpPfJxSQMiZrYf`}i!#B|{aXJ6)2Bqy~iI zKn#ZcC_0?7it_*xEn0aLIj;<25bacxs5(`x@lMA|=c^_{mfdeNp71K$*#bLg zzjqu!FryDO;kl`|p%!;;>N049dNX%#rGx&%?!gIy)BA0alo2mWC#+Qe>73pMxdf4D z0NMo8av@Dn3lA|uqEJUm@|2Zu(zL%SEVqvnHsU~cyi<+<;fXw1!%#hgpgfT+dN2g~ zPjnC+842wZ*(B*h0QsPP;@b>B0!2Xo^q+AD44(irQ2&XC3L4QU*@G1dKnL}o1Ne6w zv{3(vg&G<)`5r8h1A1swlL45bQBXwv=b*(7LlyO(Xy~F*3)6);azYu6YBK`%$Xx)+ z-DCs|(&G$s4;E=uvwjGrSeT@}R>#nMJ-(*8zhHz&=Gb8UTkHgn^1|Iu0zVfrW}zQli15B3m4b;n;sZ zl}7QfkV9>We3emj3yig3_rD(~!w|7LmsTsSqsNqOdu$VHqPmGmfEjV6vF)=;16j#~ z7u4C1mFE~EPGPAtSitXxnzSvAh(+?Z*xcYDBCqjQZ+Y8_^G#@z$h21w8| zX`W|YI2rBYXt6V5oX^5>0(v!Xb$DF#Ffsl1_!#3lEH98!C+revai_I2O-L9rKk&ihSOlhUGw-?Qq9}sQa2sn`tT$=%i5-4@s*mr$&}XezU4Dp zv9gRM1Iyhobt0)?xOdFJl15n-hL^M#WlG5fmbeD58H);~Q>1^TantoG<|5@Q9X)nw zx!-1(I3Lj}=EtjB-YlP4*2m*0`2Ut<0f&hgTtAL&IDT&M%#HBc9u$H$n)kgE=~!mh ze%OH1buho8Zw_}#4NVo$uXx7fPF4`J9MfcuL=nTgdnJC600P?N<1gd^QbYTpk-xNE zw+&2*v+oa)ObC5%&S#{g5IyTCNjI#t&S5v4e~!feb-oadrqkJ$*B0X9#rLn8l8imP zBySY}f+!*eP)XaZ8I_#Lirmqi0jHxzghC^)sXFRtvK#i_rgvbhplPvZfTo!!){>{L zMjP0T28pD1lpMz3u<4g>`S*zX6bmPKc5$x1=m1}8^eC1RnQT~2QAhDogvLOMa13Gv z)^D8@bjeE9q=~pmL~+Z?3W7`41^>y*qj3Zc6bKm5D{%ipRyK%wm<6mve~rsi7g22+ zQ{e!QTl30<w;IiryzSM_99C8G-bCedn);QPRnG&E0!oe@|14*941(R!FQ3T zG0`d!l)xX_+TB+Z&q})zakEJ#ApFCd%Pa4uYGD9~nvODTP2ugvs-@?}3PXf_$D~1H zVnqrCDm5G8O0SUh5+X57=mDw3g1id3M-`nw_#G(v`HJQ&dqw3iD8yrKD?~ug)wVor zW5YNyoy~otF}>cI8k>#5rm4DFvW(p#u90WqXwkIwL2tC1WyLw3EFc>{b3Vj|7|ktv zG|mFoLGMDX!H#Mr3Ei-rx7@gSa=Hagwn2HaZwGAvmb7wXP$NQ95+VcA1}y^>D_K=? zcFPDYm%J(|0Au350#^%}bTO^3LdQ2iY^qib-YaWzc(l-lxzb$RsdXg$@^qv%FN?5L|AMFM9a^9@nZ02XATfLVcX`D7&5@`7y-RVN9MnsVq6Au`yz^z-ITIwDE&^z4p$g_ZZ_bA68oC?w1 zCo~RfcUWEUf7Prlx8H=fhbDY(sZd>{t%k?X2+gXEsMi}zE{+XO&Z$mWBh>Iiuo3|) z#Iz&Kplih|x>Z!UShf>HQ9W%*WApgh5m z?bPG4i(I`52;1gqnEBY##i6MCinCJw8|-MzK?_{?h|}nYF6#rjw?ei-z)Ci>q)Ej} zE-RiOf^b-=*9QajBR~;W`@0e4V_Izb<~dMnJLB5Me_AV1YLM@L*BkUx6MvhSW&lui&8f z0j&a{Ux60)0LLB)@M|pC7eIMM9D_##`Wk`u7@a8nX_BNLKSBCaBu9@)j2@BlJi-nh zm+Cw+&3R0U?})L+B`%LlTK*(sjf?z>NTQ5Lpp1@RIW%@<1XR|ifXVtK5Lsj4v7#IP zB+yu&02b?0Kw^!D!|H%t1zL}W!rBMA3dtM+7_1?XtI$&TAh4VOSHYX$g}-uuTg6O1 z1bx*DIS45kVc4rZP^*wQ5rDjMNJt@h%L8}i0J93w6$jK+9FSF@@qL)9I1sB4b+K@c z#sFA_whUYR6yQ~#0=ViEfL4tJtg__(pdK0qt9sC;xq+(oVW^^rYY3!j3>ro}Kvjbu zFiJ;yJaDRmqk<+qYtpkOJu7ysk#J9+2I^@9%+qM3_IQY=_{5$lc&9FKQ$(u)Xr}?t zriey;uug-3O%ZK6Ae}xb2;qNeyFw%&(IyC7p8eC>Pu*_&sna>(f=D}bxp8{0rvjtQdg?kqZq%@uJ-_{v_ zEU=_eKuHDyDMqk~0!gyqk0RO(10=bjk0P2M21jzk9z`_y*^-9X(uh%DNCyBRh0Vd^ zL6G7Z|9J4DvA{>oGDIAbFz8Wi7JD?T#TXbP3!W&VWfFui+Jh#FYV~38MPs3hM!*)u zLl%u&MgnHKv2aDBF)qf!6vaXmbzq00C+vhL+J_v9Zu4=ViAFE%I7p&sIHHl5T_glW zWMOc`A4@k3QNOR~HXRT|!>~fptq#Et4MGYmd2S5&ugWnkgH53aoGzwzq z0LYb?xs8GqI#lx+wL4>>gfvJ{#4sXZg!Tb~Vx|~`5bA>i#hH?Q@IiyXKoQLjLkB%b zoO*0&`tV??aUejW0f0L2K5^E_Um5%-^70)B`f~u_PgtQD4fYd8^&1KF6X8JlG$22Z z5AZV{+y@sm(CipsA17>2ppRitp8#Y}@KSaGeInp`0xkByeBz*bBHS+81{4j`6KJxZ zB@_+O6I@UP@EHTo6P!JF#4S0Yc|zA8fP2Ql@&vcN;@Rg5$@A&pcn%J_8O4dh5IhIK z?}XK?QP4XESSLir9s;{FYSBjW>o};L2-@;sn4L$<@`MWO-IMoE*ZrkQRQ^CaV4`0chJh|h*TDxLb=vniF30NZVVG?3nhcIFVZx3il+V4RZ0 za7&gje#_)|Xg#DGT1o}s6BuK6M%t$o9c6`%rK)Aq4K9zw960qy9shyxzj00KVJfL8 zW0j5$v|$W3v`8~0ZiQ6L!*CytoI8$C9x^UiV<^^6@}G1HK7@DjTUMwmwIJUx9nTwL zqXOc;WJQaVmMQtBxpT(Juq!HhZ>+jehDW8bw`sfI74vP$R{S3EQ{+oBH;Mq6BX`p= zEaCaLL}WppkXN;=gcU@D7S39-!(58ZC}G_ind;*ayQiwXMI!R2`EH&-w~fc|<58g9 zvn?%6g3(gkHinWpB7igjLUqxMAZUw&oRbcH)We6Cj4O_%qycaV1AK0P(<&8NXO1cr zcR@ZDgLgX$s6+>ri+HI`~U~}ytpXEx@0m)*8si^q;HUF@E+KdO;)r7 za45-t(yS&(A+$$Xp9f-MHd`4#1vV%9K<2qS_$Mx{Nk;Ct28{=gM24P7r%u`=fc z%`nqI4e`B584gZsA}@Pu6xK(31A&3^g3Aq_qGlZ%E*^|pTh%3y+U9Fp0wx7xX_Jsn z3X!U{4z$=#(>g2t)1a<}qVsqLVkX|5S$91}hdRjb|YHY3xRP`zyow0Tw1LITta zysos_LMJZuM~mranrwyc5?>K^zojT~QxM1$KgpFm)Vla`y+Wr@OMhZ}FgezP5^ zi*#%EaJw51$L*f_0JorZtPhLzcl$!$JM%|MV(Qfc)54WKY?Bv6@)r20CiOG z>1P{w`kRRnp`}cFgKQUCy+k?cCqL()L4ji8%ya;zgcR&PnBbOqH|EYuSoY zoj=H|&x4@_OGYwUu1$W7Wvpyij!xLqst7ZbICK_lnW90|Ls8otu7A~{qB9*%ScZuf zx+rm<_OGjAGKvp@c|P{Sa;X>bd^OajEnAIln`TO?*F69~hY6E}W=W0FgV^ubwY9RC z5W<@o@?8PW!(e^O_b3!Mb{R@gEeF81(+P8#pJT0 zN)$TwuWOV~v$I-*K`!lJjORD#ZPSVRO6II9H1@$N--c=?j!E~(sH`qZv1C^~2EU>= z)y~%q?1uXJ4scwTQJH5%N7U&RiV6E&i880vOQJZ;Ty+@n&8UcFH8Zpkt05h{INc^X zD->JByIB{daO%dCyr{%nB|xG%5rdV1fqoY2moq8pdiYNtylF=0EDd@jO zts5^=0NF3ry_=-=#ahdlooF*Gy zXg-4G0kwoWA`--|b%fBX7XZX*7$6|jn=^SV_m(rZ8aJ=BHY&+chaey6svg=DZF9n? zhTR;*@Q6p>@=%DPgrE^ z96_^Lo4~cI;o_?cChOcTk-1^XO@u|1K)wi?eVef;%Cw?9vkOFJyT>hOT+wpP4izX! zpZWw^(^M&D!=mL0EuREE4#Yx=Ohl3J?AIta5W!d@MaJ!h$}#dDyx-^Z#`1=LbP46Z zL}rHf13ouaKUW`B z_#6)Jz%BJG6y6(vMGDO3s(rJ`W1@(@*Tieq`I40YbCHyEJ9x~oL(N*6EJ{ENRfO@| zBHL+_%L}Vyv#BUNv_3BuU;c6N42V|2?uIN!#!keG>fgU~EYGmBpxV$v42TuounM~> zTep6lFkl+!=pa|n9+5m{2_3X?8K;kAiE%8m`zLORhwWt@f0oTHt)O-!D-7S?8uFITz@T!Jd)73{{ttmuvNC}88u%$a_T}vE}Rz-<^ zb*$ipMtvPes5{fA8tM)wni2wB3{Jt(t~z2FPbzZ47VCxmfgZEcHL{Y?bY~^uh9Et7 zC0VwDQY8=!0rfdB0JvyGE7Q?TOJm5;xUbM7Tym5q5H|{;@Xg)N#Cq*gp}by;?%6U6 z(4&jJGnaY5f@1We5IpuiFt#2?yKb;$M@S$SDVN}~-OWpu37XEKEcq?Xj(n^y=fzuA ztgFpS^4#-U6n_M(SoD;dW!6>GVKQGllTe(q0qO~gooX59Shk?Ktaz5^b5o_SlIe0$ zhSf~wc&9(G(#(?5Nf?5SC_ByY0T^iNW4wSe zP*{Z+gG^*qGU`&vEy2OyRYK6r4`(0VpTGb5C3&T|r%CaEVUkqO$3=7zm69f?h;CU4 zQiUm7%T_C%a8@KuNC_({UNUPaPm$<5+LslDL4dJfBklm`K~V#mK)J(eLvogj5>fLS zW(6Y>Zj_<(3ZrBak>^;Ow-V%ru`OC8ae*7ig1pWuzRh3;$Q@o?BLAOUT)=4^t>{7>?H7Dg+YW=*i ze$K6*rTu$(`j^w;5R1UQTC?gENPOpd$`N<`Z{m)KRmBP{(5%a~gI?L5+m0MLyt|+M znq2XmiF!b@*jti!8rT_q%7WZc-sH%^`ZT%W%?f#hqu=VfmD}VkD8X+<5_OV}(7auu zmzDL@3lMi0plMSHQpgO@TC{}@-#!#{gk&^R^W2t0xJ7b2dKU};btVp+l z7w$Up9}7i=HWV!CY_(!ZrG)6~texsL7--7c6@P{cw-h2{gL!6ZBA)eeJx;hx^IlCL24 z$-7k0XWNtq9WTybf!F;V4OBtuPQ`py=0LYVcL}OSYDSXxZ18sM#Jb< zLZi|?YT2R#N&y|)Dvv2GGaAN2DMYuN-96Q1Z{|w)nd`b_Pa!Mf>A%G!m8j?H+t=TM z=|P9(){?F2Oy8jepfke|oDzdbTl4fUk7^1x)}fKBeW@#u+-;byZN;pkElQ=) zM@&bZAOiy%i?bf0$L$kcwkCi}`svXbSyD2GsA8V(^1}xK67u;F2~*b>b0RT@fA=&eqIr}NBPFy8T5Svv<*PH0vW%O! zt5>3Z+hb4I)D5T+@1CP4(qOH{}HCk3RyuM_T$m8z)D!RUQG!1T6 z`(3%A*9}?vSQuKM7+U8&9meTumrGrzy*A_aRPd)K`{j%z$wJtE^gEfHaU z?)#XD^;a0}9v=8xvDTsS{g7%;d^#blsPp00=j3R3z2|CNt9aZ9IYvw$vOqcafPQo}A}fUfK(c`a@tlzo z@(eK1LxcEvt2%Yp^;lZ*FiNxra(diE9vV~#g!qybbw<6zV0b}3eD&&!FTVJbIk`|y z0{&(KICa(9TW}lrv&hwVvj$>u{ybI9z1V#D$J`)IHjUN=*C_?LoDu0Vu;eqcnUQox z?qj(A>g%nfFFFKaV-oU*r=q9C9g4EkgP?sQT$L?0bq+6T7Y{gIpGzY^{#M6pz z^}kF-mPMXEb&Dti6IPnDlhYea=!0XPy!zmwx}ZOKbBrDsq09L7zt2_odt|*YVx7YJ z-hsNFf44(=URcNT>vwM5E~wXy)#)6U^N}giiaU|Q8)EfKlcg)G5aS>v8d#dc#7)T5 zijS54V7+eO7V=K~w8@tDCmd*16(n|uw^ovorHUvrrzC;jRuFaRM<{m+JuOnF#u&gf zc#wMxjzyHj<5;e6ByL8z52LTAPQ%-vskD`VrCkq7&LZqhydWL4Uu_+BTV% zo+ihlma4(O-W4?Gi2?IHLv&w1MzCG%anZZ5PyjKnE2Tp=$RQAV?WNn2RR$Og=P5mS zs6j}@-leq`xGiER0sM{?Rk?$NXOSu%wCKF;_4USjdSkqx``L@Y>e{R;1CC&iInX8{ zTahjX&X{fDx&z3aRT2pj|hjt2sc1_Ayo0e}w%|LtlD zZ;{vGFPE$ILIz%t8%MI**bI8o zLhg{1ubI<#`ykjI5*8K)L4*#PQ2;}TcyNRsdrl=yc(Fdra;ZP=n+G~3d%r$A$`GKt zi!cPBHe)047zLz{$fIF(>TM!&A07obsNij~EE$FB*LM_)dvw-t&=~46cugi!{Y)m( z$wd0gm`L|7>mw08D@N}3L-|5hR~K)y_uw!qtPVqn*}U)fA{H?0B4yi5?3iZ9O+Z2| zop&Bcwc4&SZ+vrWR;Um6LPMb1@T$%(Sw;TPJuHsa)e(|S1c<0?0kjh`L8Yv7eHt-5 z!_i_Bv|)BJ5VX;TXXxd1QK8ji$8aEI?K`C&#F&D=w_Z2UbbC;K7Dk=@zH9ZZ;HFOV zt@PCZAa1t}I1dnhN0XgqbX@Q;-srjUt%NLD@t|r}Zgb zv88J`deL+3e@Ie>^~9M3NU2;O??5%Mt!}~8XE8CE5KqxAUx9#zD5%^8iZIer)J3ZC z*qdsD?|K7%Z`Bf9059-1+8FzntfJ~ZJYPJ2_Tc3o1@Z^{s%IuQ;-tA74<1p;X!>oD zd862SK>Kh&;X?%4<0~Dk3ctE5dBvh^5QLnpXeI&Lt1cSKb-Ez0EvpB-5}sx#s#(^{ z2i(q?#(O(Z-b9<^sN8|+GJeH&sqB?!>_UdWBx!;!2Q-|_?ieqd; zYv7%8Z+$##@K?5>w@ebrbDq&M0DD~ta*4Ad%eo@0xaFlNG<0OiK}@imVE^sp`?C)} zzCU~S#s~ALOZu9bLp6`R8CYGl8y6buz^0nUlh zRYj8xie9=~K)`n>kCYdcuI!}feWSqvYT}=7*lvcs^|{ut{y^IMuJrdA21%cTh|(Rrltk)_a)G>ZB-TTE!tke>D=NE9Krw5)hiXS5{h zo~}jdw2OwbpqgWP$r4edacbaI|Eqp6hb=3)NKL@F`{Z{o%8ABT*JoC&4Bcrib&Qv+ z>IVHl-RAVvqU6nuU68--jH}L*Uqm|P(DfMgwaHq$3|`iiDA_G5$w?~IH_j&6El-q8 z@qbt;bfJPZhN$anW()yU$*8Vo^NgIpXYO(4IZLrL%c;S5mpfT8XWa~c9R$6gLGQE| zGQ4X*F!0UoiecD)?ZO_IJqr<+`X-_&tDjB~BAMo3k$?BdT|I%QW;i%jyy*5;b&v2281M&8uc8jm=CskA@=a z5X>y1rkjy%RTJ3iuZL%J(@OTZY8}Zwa-r0tC4a@Of^UyP)$t3@j%kTVF*;&mP#Qoo zIQe?F2F2?0Vn*JJ>b!_SAF^cd*MiCSqJrN%>eAt0M=qR}$aMI+V%Jd73JXrh99Db= zfkW_4tY zYxFwrb5tA*@}Qh<)y?{UjnV8s$(C|=DHD9oh~&jOW1a7`?c44`wcgRuLpESq5Qc%4 zYE*sxuT()HnMN-{b<^A8(GA-j&Gv5R(Rpz+YkX&&+h+Kzw^?xJ9Km--hl&#sCoJNX z@#rw^_I%?>Jm`j6yz6{pgqKQ-kS)iSZOEC|00D9*M9HCQkBSmRplB65p;=NZAx4LB zsK&ikN+V%sLfq6_v!{*clouEd(~cKZR^c&f+_2po6zCW=(*SJxg3SN%O{lxf11)rzuvvfdTk3hnAw+klx0t8`IHk7X3Fc0NULs zmN|lWl)jQw9D})ML7*X!{ZgTR2}5-oU<%g;H!G+OQVViMiaN^_|7TufI%pVZZ-T3? z4Yg}U7^?2ZFFGMwo{3k333zJg-6cFPE<}3OQIXyIZe`!;`o@sa$5L`)epoj4wo_*w zU4XSmFq$wUbY&`eN!9)34ZDk?QOED_rVn%Si;JW1=ZoOorF4!$o+n#c% zs}7r(iEDLozbhoIxLkn)^?$w=??q+&A(Q&jWsB8EW zD}}D*X*Jq=(bKLZ8G~xjv>@+z@fzRn6+D9&$1mnTiGqXwYL4J0&sAlq<{0#qpq5uB{ z`I-Do$OX9|_W#xY$j@Z17Fkctx0=sqT|#PQ>=LQZIIvml5=c%WJnjZHmJ@Zt`ofGdBg} z)qZmp8LgTeOZt*JopkkjA|ZURm8K;&OI|U!A0@Kg>c~wXMStaCqF!b`z_ee?tX1%hRKNenC?4PPazDd+shM&llg9sh<+1jd z4xY|Gp0u+)g@^}3B;01HhIjz>XzTG_Gy;m`RSg7WMhbQZTv08K_dX3Qnof$fndwZ) z4N%WrXAHZEo>UuaG`TNMz94h*>{Old0l!)ux1MUvGiOO1MFF=na6({r#+ygan^#5vbJ(SoH4GTw{Uii(u-FTZi0tiPZ@^FCe;C6nMmZ*KCzhS58`lg+s0E z6@+k%y#)4GzAFG-xak0fk9d#!;IZ}{9_zT{$x+l@)RpdSMrbC$5UCiV`V~v@MXw>U zjrs4Wl1gTyR=m-yb57mXi0%b33lVYj-A!UD-XX`_eQjVY2c2`bDGH;>e$jJo&eAuL z5xH0G`^HG5^Uz_p&=9+TIiULLoUIaI3Dv#?ErW5{1Ss$ntIK7!l&U0Pp0d)H!VC;l zQ$Y|;%oG>ohqeX6XtKf3aqQqf+!se}>(R1KZdhf!NC+mrtt+ym87&fZQTkCf)iCC; zdZ2S`5k`f1*@PNP8FXpIpmMRX;Kz3)B7PYd_{^a<;Nu*Di&>b;|N)J^+0s_&`W z>!IjAUjaIgV6r$qZmd^$9sBW-N*wCwh0c~wjQC3`mSq|-+ox!25=aM-cq%1dL#9{2JxMFnAYpb!**enN))827 zM29>-raizi;`{AZA;q73@$~uA=X(mivr${|zkvT*tGxOD`D(yU;)!g~bL!lbNW;C? zr|<&o=wF<+c75{2(-%*lpQt&%I90Q^Eb}ks&rgX59JJG-o*9!J+x&T95|>4p>KJmp ziOy!#%&?G$;_JELw{ZQe8+#{!+<;4gSL8`9O7_Hk=$m$&wsV*1JBWnqj$4`=bKv0L zDla>l@00d{w+@Yk_oGL3dBvOmeeveQ`MWpoug>1SBvXuv+ zQDR+_>+zK`R0EDbsTPN|0sq-fL~cT}V%DR58w0o$S#4IpY74q`2A~0`i=u3@yL#%# zT(5uP+XL1{eRjwKnA%Sfvwx#s#XuqLk;j7onKhS9^2D|jw>)*KG`uiHH~y?-;Qx5} zXSE80iK<}aeanhIWY08hx;eY;acocOr}N^qoMiVT-3&xR5^9oQ0n#4z z$@9N|5AFp>P$EH4lINu%e@Hwc2?7_uy%+cG;|Yucfxf)CKG*vOz`AUyQ6DBqXS|&S zx(t}@s|AS>-7>aP4VZxiW`g)R-InH%8H>7+F8gXWD6I$aZ23~ssJl#wryy!bDb#=> zs2k9eHYrhxIi2%_7i&Vn$Bkw6SY^?N{&yQ|OvO4v$s!RH3)zuVQmmvXmn-tTw=Djr zP>XekDy&5?5W(vv=m0Z1u;_GeLN!f1aynWf_$VaoY8j$SZ0VMMt;!VsKBQ7<9UHSdo_`1S$G~x2 zppV05oVuze@S+|~GGyi4vGsn0###}d1fB2Lde$S&dL&ufxcZgi-oA&)xn3FfNb+q5 z*%ruc`(@gvEYA)MQ#62XVG?Rb$fEw$5F(kvq*A{`@=K#2NwkX;Lg&w$aQLimO_&M# zk1DhX01G}G5A%+==uPqnK9h&niV(yj{GNF_X7EW#98G79Hg)Hh{$IIH{F*AR@t{eq}Kci;2+f zotS^jMVWQRFrqWvrmjQygJdc1+C4Q<6+tN&5^&@U}DVRxAh%%5>8fn{N*+Lx`rL zO5y8h)3E{m=|9Q+i|OzFeR|pj@%Kr(%{^PL6Mm}`ddnflF7R8A2feii>{bP^)za*J z>(Iksx7t_#5Xddhy!xTGHs-GdXUk-icVTU{blhzP2;FG56IH7ZwaX%EZ>gtFy%cI9p9Dt(IDaz0((fm$f5Sme-oGBTkkt^<_tlEH8rJF8EkDLETGt7i_G4 z{Hy))S3HON-l@ zkokJQPUC7YZGAO5&0Z=^vqrg|yV&k9$ecmwE`TFwH;8CuFN&B_Si6B{Ye#sR&7OqJC%ni_$btZi>LMw*ec?`PzKt2<3@ACQDe)!l$AqvWofvxGs` z7$^-XjaHatZOU=6_qkC19Cb^Zp~ggo>*B<$q+inq#c;TDJzRa8b(#D^tiB7?o$b^{ zZhYRB0)R6ZfZ@QtwicKGm{*|mT9o90V$KY3nkl*E`B&sJQzaJEDmFgTYZ8Q%&?<3g zUK`5H5ex#a5x$39VvcE#@AnCNUwrZ8m^^vEQ2Kvau~1K@tvttqJZQa{-D`=+q!hBMGwRH)7T(?#A-zka+yUZ)P2w|0 z2N|QGD#9Rq-lOUjyG)%Q8Y+pmUWL9|7UEFZ zQA9J6FnZ5y7LaB%YplBbtTUCC1XisgL6xh>E||SX_6>`XB}MXI z4Ej&8*y=uX(a$Ik)BS;pYOSZf+|W^LQEqZiy}hSfI&PqL>23A${q#fVmHX2Cj%VIj zuI4V*PlMwvsw*DAp?9{Tnvs&UObVWRivAG0`q7uOrp5G} zKs_qg{+i@B*-X%fD*L6aD(XF{V=>(~^c@Sk2JHGz}VnBU4IL0T7*v18ZL zbn@Y~h}v+{7;2AB(R*Bj77r~3$cI^!L!^yA#(k64^*c>!&lq#05 zh*FX;<9;dSEN2?0Jo{w0LBEb^RhFIgN1nyqRNpXZ!m~{U()?zeFqMzlS3TQm4#rircussAYzYZl(U|=L8iIB&SGOBF5kW9jeG#@ z?~^?D2g%Jn$Qh>N?XHK^d>N8x6vmwdsA9;i+zi5;N}95Q$r^gll%UjqzRg)SD`<4* z^@@!8!Dp`@?BVdW%LWlSOs-iSD25y8WA4U}?6>W$eABcfRfR#CxBU0`sJ5j5+MmCF^hP*`JlI9Y~lCMpHIpEoV|Vp;g5Nt5IY^e zsvl@H>_NEH*Z-NleN&^zHQ$6&V@1wJ;IO}&fB52yDY@Vhunf2Xda@CjHe@&x1)+eb z32Ku6Y`a2eg#r*aF-?FkSB1ga(W^J%QGqFl$T~$29u=`sf7pjg4TJ$7lZ4+fvQXbh zQRWrWsR@wvwrP?fu@m)LRIK=baY()P$;MuSr#Cc011xjbsEqBy?gNADBt_93kH7SjtJjIIA zOcR7*>OM225JuYDW=iFqJ|h~AJcXWsETILzNAFTf*KRgfZ(on5(<5XdLK0O_ zY4*JS-KPTt80k$MqUPHt)<{gL7waMhnhIbVz%j+OjFrdXD z61hLj8LtD&WxRHj&Ul5&XuQHCHC}xlKD-)x%)0q-_=cq-XIxZ73!q3LND6AdWg=7Z z+Aw9@Ty(BPQWn^F$YOZisUv8nLn;Y1Xl7rqNe{bM*#uQDyx3icr7vJ14L8?BiicSe z6emy5tAslnKy;NAOfG1|p1BqVmakaKqz50~21DAMw-wD|gUTE+35C`{m%;;;;c?5FGiS@y-NsSk5c`g(m4#)h*Bc~g0xMjL zQW~F^Ew~g3sn6V9iLmtrRX>*c2qGQKZbZV{J4|=C(CFIb(jATL9g``$V&SY%w*eP- z4cL`aWvq$dD(?ey!poU9JYs=%R8@Uwq+Nxk1`0y+Jn=f;gJ_+Jm|d6igsatTu+hKm zWemQmj_iFn)THt@wAlU?%M=H_WTWDao`yd9Y;G6re62x}tvL z+bnnAArVaywOJ3tvVFgfuA77-?A)$2t{0)D2!w8+Rxw2ca5mhDqDU$*`7|Q{5J2z0 zBC`z~6PhIUJ9WJPe{Ur-8^E{hMrX@s?`+vUirStGkq6k)Cw--_>dOiuE3kGC#~YGI zcvcjW&&$HUWQV{9x_#)K%LXYodjxuoKDq-Nn-~H|&Nd6MP*)pwn=#f(0q6txqo-~Q zbXY_s)S?4^+{DF1(?b8uc$aQKf&}z)SG6`457`5i@!54}{#a8btDkq3Qu{N}ztg!W z3!B5>ad>Te$3CVPtj)k?D$j3vvb;;8)yw5G-x1X##ddJ2p{2~*>gZG(oob(lQ|$_! zYFd&eTtQKt2EVaosQgyCp?-^tn$w|ryP5XoXySdEj4+$Fp2dyI;JGkhGQ!epw5SR6Ov@}As3 z?>W2MYk*sd*ahP&e%HLVOW{?fa=lpUTJZs?@by(YU}0M$WglUeV867K5=Z z$R3zFWJfn*kN)B67(v^+b{`&Bme{8Iu%TLn?81KKV^hHhX*Ea%X{s3k&4#!S9p_+( z^C-Z5=x7!}&e;(6p`%{}v56oJq@iU5H2Yb*4;`U-Z9IwFuEYQ$xb|(JGZu$1>i)E@ z*Q>Xh@g#=qRoQSK9vw}mM;Q1~9~jDFG5I55StR06FSYKYR0XJ2wJFMVo~E|HgWQtX zzk{MO**ZlpcT|dMRj8_XWCmOnNetGBYzvAcJmpY4AO^<6RH4XI{BauPWeCqOPp2#u za(!GiqhD{dYo2%(!Y?HwQ7I+Mie&wWI#)%xzGU|LYEDY+E(7yvCDVZyo3x}58WP8q ziD>)TJ-Me+^^cegSF>V%&y@&TE?Oe4-v%sJ0(#02Cf9r?xbv;JFx=GNH`P{=Gn@Sp zNPbgfv)&sUI`4CTX&iewkPScn_UB0rxA64GDg5!9XMcY7v;D_!pFMl}^oPH`{^s`j z@^Adv&p%{kdWZk|`RNbr@^68SJbU)%b|8n%tD*7nte6NnLDGK(wdw*Iu>{&>ztlpi zVq%q+{s>Ebh>G1?ytUbbN7t}J<5xahbV_eS;4-`5fhmmk>1N_W!O~nvD%T`7rm^*a z`WH?CjJ=EgLH}Yd_sL>RSl zd!=<8uQRGoN`}kEV2QYq@MpVeBUK6k=S)*hWkjerEaRKBKp>5%y<&M8*KsWY7)xK z1fqSA?ej_MIy-7O{ld*gyah?OC)MLk@#a*9c7ozMhcjOjYU8*n?r$B%x0SKRh`neX znaV9Zs9HRBBQ_X#ydV?(A8YKf9=*`W%LuNh-8=Eo$gYF?I^IwtdHHbk(yVX+{YJ|I-+4QNG@ zOcNOD*hrBQmC;)gg3j>)=cp!R#KG9ZvBl` zZ$}ALmb$I&np3hO@j9dZsk<0gr?@cQALs|6L~hbi~@+)V*~O@Y=+SC4AR!LRVhgY=7EWBp+4YrjPxH8E zOo3|e;1iK8-FjQ~4%OlyBRtdR@VKN23IgnWXOri%D{Pi8SH=R<3}9^wOCL)|V@DG3a@@fk*H{jD0&^xiedWueZ78eS(+!b2#@7>^1Wj+jC{ zA_c0Q+6cwEwQM!3P>-~tHfWC$TmUScvpBA?XV8@LkTGd@#r1dR$AnMW)c(ucq;1BE zfkG@X#=c7VfCUdjaO}gw8G(rF5paN1ZwoNn3P$; z{n1DI&FcF=23JYBS`bx6S}5j{O{_Z9xA@x_swo($doq~r9@KdC$pfN4zpvdgkv_j4@5idY6 zClV$sXR|s9h@86+&A=~KmPSxZshE;CBGA){0okA`1IilKD677P%#y_Uf_n%Wrhp8N%DTfwC34MR;W5S z6Xls^j7EDgmKzXuFb%z}Qe&)WKN#R7Dq-Z3mcD(1AL61zJh;~)4)f=2K)2N|*U1-{ z+@rv}g%1R|9a3ne*u=OcD|)}XrMcE#Cl%5RbZG*gn>-J3i3CIx+H0VvEhJynUYbYE zg55$Jm4B*t4XZbC8&zab06dkw7}qT5EApF$f@757>O_wTIZy2oe`+Sg){bgE9^$Yj z8y^JCto@W~5{JT^iB|L1nz%&Zd6P1!7qk>f!X%K(oc_a>L8FQZAg-Y6>$o!iR7aJJ zJp{FB^)srlnxu_O0JE2kO0?e$a~#(FLV#pw<1AYvF)y4&g9e|VR+oBM`pA>fd7=fn zv-qn1HsxLzO@-6a4OR$rX%)yC)UeU)oqCZbovn_Cb?sDP zAq>Spu+l!*No=n<)9j$Gm;{7& zzcDyO53lAP{wv^i7{^-$(Jq<(QgD+n(kZ|M(rj&l^okXi@g%3RSfh>Rxaxgy!*1GS z;7fyJLi(VD+^aF)KsxI&Gj7%@HUVZYc%1btKh`Kiay0GL7^MrnVJ$A7=qSTuHlTV3 z_f$7_AM8z`cRL#x+_7H(HEHt7ee@>HYE?53Ds%*-0h}(}B)Ce~1^a{k+A-nX7FzeB zBoo=hB4sI2x7D;&@4=|GfEru)@}am`pPirry%}(>JiUkEe}8}f+0hdQN#kbk}(^)NXWGaFyu*T5v`aaN(3!;LU^VMnj|$= zw^;xJG*077auv}q7Vc!<8w+TjP-CK6lv#wKj=We83DVm-D1cQ%p_Ps=(34PG7s6Po zT}WV=TcZ?RS>#zbD4S}6FGW~Hy;VO7~T)!T_s@5Y}i}h?EvQ0t+BtINr6iLaDH^6k%HwR1r{?%w(B8 z*{-byuKP_wI;#t(#-TyjELC*EQiw($GabIg5EAgDOBT2}Q+=qT0E4@*H6bdOsng-D zi44;cenY+oAG*^LnpcxVJSb8;h|21P+F#goEetnO#~a51)2>!8^;AZEIO0`{(A=@! z>klkR^#6SvyT|nH89iTSnBC8nHBFl2ny0Z8c^;}L>9ziBbMz1fBJY`kU?7|H3tvPG&o$139%b?WP4VC+ZboH*er{L1Cz8P<<9eP~|6sdK*2E2cyOoV{G z4;jaUm{>*GDVZ^gnd1(+83Y#Gw&vI|D=1G?ElY3P<#%wS17zV4vO4nc7jNCF65)nh zrt17sC-~V7$gdsx;jiD@1!%&4etFQR7LUy&G$`E=ZSs0s_R<$$2(;T4?U~f-M4xV)!45mN?k(z|Pj~I(h=RZGo zGw5+F0T5P`qOqRsi>5LkowJ1eqhxYTv|eblLPBGBB9SQ`o7@%StqVB`3{_LWO?UZT z>&8&+YNzw{W+Tnwgg}bmC0CKr)mSlDw4uX>y4Cb`_U0n|95o<}k!)EC&Ahh_BDh6@ z@VrtyoN4Lyn^7X+^Y-93g-r} zXu^Pk+zU&rn<;?|a!oc7K6tW=-3u=}XM2#Ln)XH#@Eu$CRCM0Hu&R~F)Qaa8z8Chn zSqabcCWHy@q0Q~MilV=2bm-s{vK0g{CgbyYMA8( zl{N{GKKZ^BHMYVZMpQmi`&eEfC~Ys{~bU-;+DGesa9Ymx5b- za+N))>wh+QP^H5`Ov4kn7g)ZSm`trr6?m9x`)&jl1t=?s zPBW*S+F52YXSa;dK~3&ysD_doVM$nn7{Vr`$R5P+Y|6>l4NhQzaJLmR*-{&gI*H7I z+tSS0HVCtezf~t21+^)pNtoUWPX_$28l|NlkFUP!Fb&C6E-*y5l?Vegm=!=;+pxWY zS23~t=m_I(j7>dsrRt+p;l_X!qB#&qhwn$WwwGcE!tLOtSF&VSF&X!QdR%i+ zW@g>nDqs6kRFB@)6sWpqc0-*Q7}m*!rjw}{U{3~)|8l{k#JQWV+c{UzXJWx{rr}G+ zICt$aoI|qXOoT2a*9^CZSlEZ_>MP#? zKNSn|G7Lu2X_2SMG%;yF6oh9{QpOlRge+9^st`WE14(z%M(S6s4rudTs?efP2H(bb zA+e%bvr$-`g+Zea)VxCu*%O!@&jSJ9>V0I_~WFUbKxezk4A)Xd06oKRP z%ztNOW0@7oE40HvJ(2w1fBz;NZ0)Fir)xGJgiPWG9`_e{Q{<^`=$UUM0jAwSNU(n6 z<--Gy2M_}uZxX>CeKiq?rUKzH%L=`@rC(2b3avdOmmEv-LE8Y&lFoTn*LSQZpwA1W z_mwsDD^_69Fev&UiWN~jEt7&~Oq42FdzaIPoP>Viny~k{5yFIQ^>|a*LfRIO?1c3m z8*G3|opBqVRFbzUn#bO=$P`=2lXA&BOJjXdJp3BkyHrr*+dq{K68^p-VmFBT&r=P! zZH1>NG0R!jQ_o9e zzg}wt9;(jb*yLdK)&C43CEQG8GW8sGU{MUxp!nZYzC5w8h9|x^| z)1z;{y}B4mCjE|81mW){sRBHH;O61~DBU9%l=!cRW7^tpl)bxX8L`@AdcoCQ7!23e zYt60o&G{uUzXqOM+UH4B}R`nc9zB#{~l5fpX z#)xI)YI^*7O~Vr(lCVKrcMl6e$a&r`12)e3+dhYnK`_5Pq5oupD{5z*iWfI5?+iEG z(4_F;4f`)I^bbODfHI{8p~y_4GNC@d5n6wzpZ zE1pIfxkgaes_UJGhM?1wg+L8IBvcr*wlEPopX%RHB=s7KGf~-Th>!@DXQ1Y%)1xWm zWE@S8tRj}!U`cPIMmE)*aq*j)3vWuuTbtbK*Vy=0yk-?|v9JjKrDw8y}kSNrz^cjJIw7>F;^he)I;qS36Xl4oHBE-TKl-x&Hi zYwy(eb5}$GT6AvN4gP8$-#DRGKbw?vf9L`nYk*%3LXI{NTgujhG9vAy>eR{2<=MsS z%V`?#!=hLg=YnVc5)B(T+V&N@cI*!hl9#E!TLMNKS z`yXqH)sPcSv9biIM^m4|>wYFDU zp|c{VAy7${&r_n-X~MI+8pLQJOp!!J^bNayC8PfQf=%hL*9zF1`xk?(%wSN~`|K~4 zO_*R0_b~tD35>2sWB;QI!E?Ea6c1y#@MweEX5?yiKD%oAMRtsz4hW$39+12I zB3;51MspZlHCoa9tSnX+TtN}Wa0O-SO(zEfk%Emkf^bb}??SEHIRDGpHKC=ZUSVJw z_tXsPG0rf@dGH9$dwYz}w(uCEJ8bxT!;jTtd8fq=ek^~Rre}lwSbqQ22MERAQ_{eW z{#Wr`@r)Oo7D5j48#LakOem!)M2b5M%$U5lZp9lg=3One8?5s};S2;czsmHfq_1#u zGyeIQWbDE8iQD(de;BhOpnW8N`YIT{yoM^Yh_A+06p1WjQ6Fe-*p&PuobB;{WE+-J z9O{3SG(qycIfR+7N&fKOHD0qFt*7BlWiKgJPqY`!%*YB!Go|OP=`T4lPR=9pw1{%A zfA8QqFiDxY>I(Mr7N_G?l>_7J(t(|Tu&To?TITs`e%$_F!#(nfD+tZ$!=s0XxiYY@ zpt)V^LnqIFqH!$0LK(a;nw0EK7`IRiG^@PWPm6VKu#Rreub~x1{`iMe(Byvi;t!|Z z$^WomRCwzv^P47Dk=mNmohTFpz%CWpn>!w}+HYJ}@)V{Gha9vf@Z z+mJB@FFf}pNcY}ma+v9XfBU!AApYj?FmJ1e;co{y=<|Pcb#gO1 z;oMLalhy=&VN4Rbcr*Ly)!EmVuYhhC-QuxG7AvEb?-KVIzH7VoSMRiCp-#JO!(Eq= zH!o)mPu-SBuG@mL>8acI=#(uwDey@4PN)8KOShq+RNZOG2_^k z&F5{Uh{8Br>#L@OkgcLr^FPnFG8t^Ppix_GBpXLVQwL3}dwn0S-q2>5DJZ|P7xRP` zhDC=~uWTkCcaW>^K(b9rLY0-eCVb7!Qhp51yf>}Mqw(#EZGYdJCF&(g>$&eRXTE-( zQLUMsIm1d6BHJOertQV1)BYNsIPWjmR<8(*U#?t9fHQ1lqR6aMsp(SMV~_rl(&`Us z(%*Nh=J9|;HvO=r`PGHdFl{E9GK-7^tTM8_(jHx+$5$6SZ%iSJ2O)

1e*P5kX__ zZ6=X+pGH2dy*Gosw}H3UwFmtgadl`NxCQ-wkjn42u?0V^t+%_B?5WC7iB0clcB;y- z_xS^t`~aR~UoeIYJ#jV1p5!;)cIf)d1hf<+oOvFBo#(M;*N)kOQB6DG| zOksuYaC?MkN@fr;Us=p%qC@=-1lNI2SN~(q3sd{qRET!Y z+J09_Ry!p%8__Joe8sVgWkf^;O)Rt}D;^m0Ym=v^r)YaT{qoB%2>^t}EaK^ABLtuN z{qyHj@;_&PRWH6e&^cy zXtC-=Sb}oNi-l9#mMhB{Y{3z}Tv-5ey~YyWQ%N4oDQFkAHE%G?624PO5*WfUN%$Qj z3kA^Q{#+7ULs>GVRen>w8b4I8F2%e>hz{$F48XK)dWg(swqQmEUCRcNBQNOL3;|BQ z#y794R&g3k0hyZd&_rZQn6)1Nh;W0ASX^cCuJay`O*m#$gwffzvQ)*_Wfms^g<~6| zeR96l0><{5)C>z}9aMi<&(XUnRK_jp_TJTyw%Z>IXw{g`^?srmp=BW^P=?+HTTjW4 z8G*XW$58h;q>#DMptk$t%)VPoIi%L@k6BB5RqyroGmfIUMb!%JYDQCbthuc%aNMjG zsQ5Bt674Zzpiy)Kvkgf2SJa`kjM2MZ5xQIgy2yRVEM=NeB>bOs4Z{^9*O#v$1{_pk zt%?;a!{o-Wn&5f?)@tySkV_>BsOYC#HWQ?(!r%~0GMOhL@|P39D-x-}6v<)&FOe0a zu|=QOYEhoEcuL5*-i!+#(IN=UP`@3sxYT;J$q`*KB4o_4JpO`2PJ>1U1#T~p1YGxU z&I?K98kHA2cbZutK0zRbp;w632-#PDb)O6dw1agv8-Opl!JqI|Q1P}mml^_U2Y@*! zus0V!LH`I+A+j9{0w51W-&))h;PSFzmJf11UQH<}lJn*;DmoeJa*C$9z3KjPdN& z6iUG=Cks<(-vCm!bUd(io9+1MQ&}POpc(seOc;$;S_PYck`uV5w8CjM%L_f& z%2&22cLq?6IE&Q@m@s~qD#l8gAfwP0LC4IT2w-c2GkeYc4C4>g(WmzA=QJfU5;5{EC_4%GBERoFw%Xk3u;XM_^vlqFo8qWFLnX zb*L}JiZaLULWu!ATBZ~NGLw|bI~G@(Rp=~TYvNVr32B8MMB&kOqrhgkA@^uHJwmzJ zQj{~OjX}1B#uk1}bA=WWtqXxARzPvsVioR?!;1#sXt~!e_UD1N=PtXm8K>DzLo7Kv z9n{l6Y~a>zA3=;t19Yg0S{Y;z>WH7h1UQ2TFwCgs51^`oqz1Ja6i_v&#VYu!C%|e4 zOL9jrmJ@I_s8J_4>uLeaq_qg{pf=r@x=L|V=!|!ww1)Y`>WHqMsI5WG`XanGqPPaN z>xuXZ0Ry*JXb9NWu!2IuHm~q=7+%fjP^a#wY@Eynd4!!QfzxR2FwZG@Z3N!H#Y43Y ztVB{4RSDFcB`01mFl#`>O4hc?TAB_Xwh@WpLfE`ITtE!9xDT62YF*#a z4A@&lB5-scxYPt*Lkms>HXBlw0yE3x6MLxDBp9yS^G|O>o}quqQZAe6|*aE3_y$0Zing9@ck5)SuuJ{qBdSt**V) zqOy+T2o@Xegcic2EY!lc1_upy3Yq+2U^I_-AcOXMiXClUP40eUrJu2?qBSPG*LVqA zUMBtuoN!qHVYolJ)k<6rEGF3z@h6jJR8Ea4Op9e-;*4NGf5(_(&y-0iQ6p%)nB^OS}ZZvx-w{%e3qDFtM&{%T=^_9 z*19sR%>$&iZM7Mc>Ncp!K1&SD)u0ypEitZjWl+-|OH6RS+;*Em`E!FUQhrMeiGtk9 z_4UiL#5n4Wm6{E!%y2`8Tj9-`X6o9z0d*}lFXPOqj50zt0Je^frqiP#vr}Ogm}6U6 zEGB;>EQ>^JbAT3H4(y~>(rX)lYtaHWS=oAq?-(54kOA%_T5U%5%e@umT1~F1{s&aq z+Po@A5~DBI8RH30xyc68?3Ag2euk&mz%2?Vzneu*T+2jJ^5xP9fIKs34;vpxs@RMP3$0Rt=s?!|E*lMzuSStMk&} zCiVI1d4FDG^U7&aFqwTt{`cvRzx{dg?9Wf1{%|_^_rLx2>5o(R<2TR#{Oo7@kKaCf z_Vnowe|`PU?e*o~__LpX$jbB%|Mm0JAK2yJ0vmbu?9cz-z>YS~1w1PzLQZf>zk&$J zLCslm@3*gO(O<)}3_BnYI3#Fx|A98{ZpXh~RQ|61eU#JRaCgh6;zAkFIF@}GXk*5g zI?T)FW=xBal*dobvivb()v$znV5?)G+GiZ7hBMtT4M6>EhAm;M1%Q~^v@L|#62SVf-Q44bu+cK+`WU~%8ndsu z;Eq|$;_61=5;NSS(4(D_Ga^~Ub8gBcQ2ZTNohf`A%5Qn0Sh7H^h-P)!Wa%7=&Eas5 zE~7dG9OZ#4M$}4_NlYaByYWYdGJG8Wt<#%E)DV!k8p5!S(wrOYnaAK$?0wE9EcO|T zL>BL2xY}x?R++#Z7HndPOIf9=-_wN0yjWKeVYt2Y*`v2Idwto3sp>f=G|#1=(TXXj z=f?wP+mx`cirXgTl4p1*fR?qEn5>Bu1ucSC(Do4BRb@Y^KN8p?s%(W^13Zf`-Eru^ zosAG~S`tZYp*{b2s?WcX+3i}!0N7X|ND>}5przBo?N#FwFTNlvQA#zv80J$d1bGf` zqDje~uM3hestWS_l%zZ>i_j7U&XMI0UkuoW;fIwSiyF8G6!^P3r36=QC2(NY>n#@? zr_w#G7|}I+sbryL?mm+=`@jeLRr%hA{!xngx(I{nFwZti^Zl3kaSJ}Mv)>M^pn(sp z(E~eP3ei=TQk34J1dXx4)AodHJql3KTK{MPk-)s|ohfLicZwbj;Gzwi!p=wt=)c$j z!Oz>`fnb9rTXI#%Ar!$Env2tvN)nQ!Wm51wsi@-}#EostNJ-Z_LLJjPMPwO-bj>OE zGvB9GA6QlZ&HzNvvz$s=kv|e?O4T!T!VntuNFjrbrsblfNlgVO_GRD2S+I;|Uf*dG zMHe0?;vqw*Hb$2Go@E=Wk_4EHzNXO~%VP4avc!qk8BGoFZ7#(<*NcoVKY>8MuiOQ})~0F2ZR_wgTvrhm!rBNn2I>NfWs#Jv}~X=@mEI-Ol%3iukoFIEOk zG+XhX$1F0*#zHZL?jIe{*lwN92fXaGSI)|L7?RDAoX1Tbc0*&K$k$M+?1F*P?#kLu zx@!2CO6u2ajAD;`%kp4}gZZFSM+j`_I{E zLS2am5)1MGAqV&R>;_(aWvwaBWG!j-wr7G~idwA+5Y5)YF?OiINj+#CBAG~he7hCsmJA)6N(XTL&#kr#f~%Eryt7vHKr zyOE<-pRgi2iCDfmF|F^;j1EkmRAjsmG8~TR$i9J6FcL{(uwul5oU?pYH8~jd`oQB4 z2ssHnAcr`Sav{7fi4DkRA(G~{f?se~ISWV5dn)REBFpH&knEd)gMMb8wFCS*`0F2x zO#L|Pkz_?r3dknZx_Jf8=Z>S1mc@!?1&`3OtNH^DUDide7I>G){R> ze)K27BKwbIpjUO?wL*2*6Pb!=!mi#1oyyF!&4by@{OI5__Z&mIFv9;@Egq7IxZ6=M z;)6F&M9!*5*y4SK_L3ItfriU$^SmV=jE=^!WJ*~MYF3GNh-cSV^zKi6yr{V0!r=VJY|4qaF32 z&t4BwD!I()Jn?&4n+v*ng}hAYk}&wkNz9f0Kbg&5fhVQR%vr!o1@J8x{##{~B2(CZ zuOg!x1pMv1LwFWjKv1WCoOz-*Se;hsHyCU*XXJLpqq}Rzc_O6#EpvVfHsTC;FKC^a zVOq!+p3)_il?Azy89!^K~mX`0;#)? z?rk!PW9R)!V0S;^^G@_1e!K~>ZRVB@$2$II+)Kb)a*>ybF9^*c^WOdk{Qf>}C@04z z{0#0U=Co&b1y#Ok$>y@F8ctC9v}8UtbrQC`ZH5%O>Ppb^HoQ^T>JY`Mf{Ig_q^i)s zlhZ$Z@x_!}a0viIyh6~fT^3X_P)0G!3C^VbY{4k;o~l$W*;M_0$c-vg`({t=m@O5W z9TgT}*y~+r$w0sIW}w^L@*a8y<(&`A#qP{D@6RLmx9jGZA2fS>K+d?E4eo6-BvX7l z2fTN>y7I$&GQa)&ubCBo7*vv7u}A_OZ38p912eV5lRAR3vJaJuJs|8hEblrv+j>AH zN^g#JN8WTmvj-K%cs+th#%qXR#%$*hy7twIZ00xzv^Yc-hr-VtSR&)KR~GZI+~pxj z0AUFL!P&_}ifMXtlDG4W16pj!MDEQl4rsPA1Psb44rX*=xdr`M!vUEG2TLXh%QPLB zUs`95`n8+k@CQST>4OXzNA)|c<{4Wzs#m=*@tAc<^+ISK_{gVY61GnJZ3y^P(Wz>* z+3DWSHk_!{&sNbkKGJR{>gYQ?gzxl_(WD>Gix>Ygy?CloThkijfO&wLCgvds8Nhw3 z{dabKMIy-{bC@OyL7_M5*%)$rTQ%CE)#+$aur%+%!5y7=pS=_BAP#|5nbK^ccQCl| z9I5K1z@`wH60=uPw1VXiQrgOl|D$9fMbc<3l@I7Ef%bNw4{8S+yFLW6FG>Ej%zXkSD2X5B)wS`{AD0K#4McGDH)dfxT;Ogj0Tk(5g*r19ps!|=8V zkth0`$Rr+(q=`?MvBE47Sbl{dG{Z!rK3jne429uaU-6u+=sg#u#Fh&|o|s!tw5(hU ze|c|fnXWPaftspi&Z^ZAU}TPkHOx%_=YOlYYPxK>{72Rp?^Uj4?RVZ3v*s3X-)L%% z_cOS)tKMj;9AE~gO(yvmx}#rj?6c7RqV>kg_n+`+eRVNBW&q!_(nnSC0{VCIHB~)q z@HNfo5{(Q`XV+i9ex_FnSdXqQHa*a;wuK*fvwvFS1_=g9->o+vfeZ)8+_dtm51Oh) za$av1tia#^TN_7N77`*o@6-5hzk&1z1=PLXjluf9FoC&fx#4ZI+a~ltlXH=3cH9of z)|ieb=QajKVICj8W%Y76tWD^O(U>G+NuGXx_Uc(nS5Z99GMeY>DfuxY-=Do|w8xbR z1dAOHIw%3xQlzX{v9e8U{^>u-i|LEg>8Wn>##+DYSIGT~>GS7Pw7S!|xM$OCHs?F^ zJ+OHXg)YFFr5a(fRe2&%wCdhGusP|f(&U#^`mBRf(Y^>Xaf~7eIx3mUMW%ekcES#u#+f@~Q7L(`MrJs` zr3ZQ-LcQvCCbpr8VXHWle$Uj#sK~dOVuco@^Mq!1elZ?+F4PZ1#jT{7(wc+q=?;UP zK}(aWscO__xP9l?tx4P*F1bz^Rg9$cPHQ?nyxU3oE)_#>9#+}V8aLF=;L6Sf`jfx{ z)YIr(iU$SDP>@8>*xj7gMTDn&nrmf=_6LPvx;|>AAcM6$#<)XhQk8T3mx2QbkZIQE zjz{>hVsPK8{aux#?}3=JLMv-yc`@#HVKzfL;m@es@Hz1b340Y;DQ z5LdZfYX3d4J4?4SF%2DRTahzxsj|2(7m^8;@x|ith~7RT@8?)ndIL(e+<)R%$tUEE znCXEXWNKe(F|p62LIc~g8O+@A^3Nt3gkqLq*pA+7Lh@x_*4%z`+Zfp6PwEBB`W%Nd zJykI7gG~K@&t`S%&rWmy-?Q1o$sVdFpC-xfbHC4|HG8x_nF1`Zy1Op|)gnxRfW2|@ z!XEsmfhRDjB80P^wn)oe<>U$IR8g&Ly`^Qyzik4^KQX03h6+g%T-}L>~ppJf=l$bSxxaE}8Th7W!Q?JyVoYExs~b5FIS8 zQ?pO$d!Cjl$;x!jq#n59X_*u>W1`eYqpIPK0i2P4GARt*Qbujzen$a`1yk8kVPb;7 ztjV)Tq}|K1{%kwl`r}Yip6PLYMNT~=?G8=J>5|>@BI&SQG{||2sTuBBNWV@B{eFA@ zdv<1#sKI9*EGI-qw+h2s?MsK*zAk`k%Hp^HFN$htgI&0wW^X=A1PfZ^;?3qQOL{-NT?f=koxT1P(wf^S1Fgs28T3;A@IL^JrS zVi{1_Y)V3Ls38&v%8}ca?wGeG0A3^8`PXJUTj&}Aty%10DY?&D^Tl)pHE*vL_JIMVtq`$5ta9%**KN`QSxW^lqYa7 z+POmla<|)jDH573;ryK6El(o3&I@r;B0EWSF zrl#a2B*22}u%JAFvK{qvqTlNopg*5CM&7;d1J-*~%6oQQ7kmpP<7JZQzawnGco^Kk zZUR9Go;8X zZ)5)!o`6-Z<5neXTki+Nz_;A&xSdGw8+#njeo6j7ENdtPq61rY16G{P^E5XRREVo7&or&p1|hgDl>p5XHb;s#qWxo*Ct|Ia@%Cc%$HMpmU36D?#|dFe zn~F-Q6%JSwtYuC3y4b0|ZZa+Gj+<16*+0%FCBu4{s;_)%mElx(@|c_@Ocp652T3e5 zOEaNv%7n?PBeC4? z8ho@yiwuwd96VcEK=K87 z*Y=fn-3Lo$S4^@d&tDvqd5NEWH^mh+t$D-`f17U8#@qD2A8!nSDjH569_t`xyUs0l)6X)IO$sJco zPy@g>HZvucG+NnT5v!xE&y=V#O{uI+efn*)!v|=heF|Q{8KG^IgHH;bf^YK#v3*`Z zy7h~~&dQA;qHvGFw=TmE(QEibcP)lSr>|@(zHBV0on>a39cJY??F8>>tM!h@z!zp6zyWJY^ z+jsr018?8seY<;N6!MK9TQ__ER`ze=*6q99?FkU(SDkO+&iD$<@%RH)-f@2^=ami4 zHVj~t^;bYSo*9J{s^k!BPq1og^{noX9g%%h&mC^b`Z0KY^Fh<5`=~h|lmGkwe)Vtv zVpp`HvQQ_HjNw0%I_XfB2~*QGO_P1Uae8`s`sMF`kN^Jz{{QsF7x@3v=U;sB#p%C2 z|Kbl{{{Hm$&wr<%dvW^v)8G9YIX%n}{a2|9D*x^Dlt~`tp666S7YomQ9L#C;|Ig(5 zb>&Nu^kItrlu|Ki>cW6)rCt*gtgj|y>u>j`(=T64Pc6u9+cT3$#wP~SmS@XpBqS3` zf90pPmG`U3Wb&)p?f(^_IcM(+mO;uv1>*bsS9d&%zarSBf-xJb8a=3%^sfFH6Y0B7Zwdk)c zH(zmcZp@QX(qwB4$hKEPLVfVY_oZ!H@oZTpRBpcfD}q%$zpBeTYI$ZeNpxEiL%I9s z*iy7&X_ed{a+aN4Uw!wxS>xvoMWHz!6JnvX$|hvYvn7_fO7sCeTM`%)o|@ZrAZr({ zeriA6JBzg;OPj7y-Fu@{#7?fYp@2SMz{v2p$W?9jZ?o`=1)z0+z?~BG?<)&e2Hs2w znyOf4{GV0Z6)4M@SYjaYaRkIEbsvq8>k3v1J78N?JY(*1Dl zo4NZS3^1n|{e}4M(httWw|LwQW z*3bT2{r%wl9Xo%ooWE1&Z{_~3p8fgR|F^4sF5xK;1D$-OpG0Jh+McVYO#wzsV{Ox{ zJ3>h@fC8kBbc1&yn(0Z7%ZN?Q$GCD*Isqld7a0leRA-Gt4awg=o{`8;Z zbo$~8zfIF;@AE~jHfYueS=EDt)ztOjlxMG4w$w+v=ey2lu8BmGpKNWWpHv~W=G7TI z+!Qs}`1*}`duCp?<%-a`GGDuKBi(x=iz|m0UuIA$e7<&P$m%Jb0@(&k74W$U7v?V; zEX6Y$-katk(-Q4OlUYjuLx_nbL?A&Z|MsLY=IdFM?93?y(*vJ zk0x4kV{|5L^+d%tC)Vhd`lmT-?pPr=zS!9$Av4wtdTPe{|6tqv+fzG<4-XF;2M?IA z`ll1r=}sCYR!?k0KLG_!+Tz9RB_#Qw>i^y!u39o*{&n@k1b*=KL_g3S;T;y{4+9?% z+3j)|c-Mi&GhzE+jq4$g)&h8y!`AvX(BbS2Igwj0~FZKs11n;lyn=Y$>Gww-ir z+qP}nw(k1JoX8ZjE9QRzYV(?y>C2l7Go=jwO*NH^D) zFI@tMd1(*v@W;x*&&U2_B(vgA?b#3C*Wbq<@3@PFyC0jHPG8g;14l#Ri5KobeM8VZ z#F<=A;Ka?!sdwm#qIl#{$jFShOGVKu3xDoge7~dQEb3JVFzoy6| z<$6k-4NHn(WeUc`fD@?OF~gO+$=A6}U5S-XRzg5X!APy}*PQVyDSrqOdD;;Jn@Y=> zq|?DC-iAwHHx8~JLIKbh-B-e1t1{^D=W_H1i1&NboG>U%nU##K|2kcFV>%6&7%t(+ zkfeoJfvddUFhH;g^LJ^LvyrX`QY)XIdOBF5o@5H9BcI=*jXJZ{%itqXN*AS`3**VY zWC?-tIOXo_Df8ZaS&AKx{aFeK?^CMvTAgmBZ}W#t)1|>qUIo8w4XvStR?79}1XX1o zJanDbrE_)zt#uKc5M6$RyjwAjiQKkpM!L`BXslv)afGCxKwT;Q;+;JH%UEnWW88hR zPma>qVJu;*(6-tEDe;#%?b;<+u(-kPh}MCTHVrW-^WOZ>KxjpKS8xhUI2 zJ6{mXqFBAiUl(MdRvId~A{j$q)!=}w?+e}{s=7lA;ivbM?-yR_M`DC;#QA{a!?}DC zkfD90QZrm3Qae%j8*lk|IA1=`-?;ymeL#!miUOYB3eG}PJ3K)glsAWXG3^R8L|yL# z+XPh+ufI|5tkjLa^dx0aX^o=X@dd=4)>9hw?P1}(I(^9I5Rvz&7CM2#$yPM0hkOLl zb0n{Y;rE8lK$%T;e+9g8QBziJb;aJrr+SfP)=2ikX)X8nzSFkDzup@>*%KQ3E<`-p zu-uN4w5@`qdVJkcrcsM+jlpY`C4ZtkKRuK#k%$$5#98~lerHnCi+9$74%ubMUs55d z6~=3TR5DpDtYO?oACb4S&Ng=AdAQ%V=1&8)j!KgS^m6HhUQH|(^KtbC)98v_>lHwe z15LJ~n7Mb5c9t2{J_`gQR(K1$8LtHYiiL|Gx{lrPe&qq|gtDcjIF`lCV?NHf+ldDoa_u7U{g_bLzVSgwF5rPzFpkQRnuq*M zP?Jh~)un!4ah8TR!lKN--rxtkQcCkNaJ`BMQh=GNWgi=_Zg78#03zo2jkc|V^va!M zYodE~Y?Mb7;H0tU%JCo6Xrl-<&p%Yg5*dKgupq}%lI!!HD<-u~L&;KbETEZvLp5`+ zQZE~a(LA?&`Iq)I9^i+8!Mx~Cy=RlnrRT>j=yR8i)#l4J=kV;*7tyzPS>87UXuQ?c>~l++O$|XEQhR zNHE#CN=#ZjufO9uk=75`)z8!#;OlN6=JzV4Kv27!V;OQ0!GMo(LKa50$F6 zyI}=sn!PPuKsp>JrZ;atv6205WR1pRgcfyhZ{A!qx0W-Y_OgfZe8K;>KI%Zjt}36h zH(Yvq7q8?My7dWLL}|=hG^ghh)U7`0?Unjpr%$r9y-P_gM~Ys^_bl`pA$(rB3Z zdT_Jdb3+i)XmC}RMQBx-d`8{smn=_8Bb<7J-Q{z#~h|D#N^HFFEGUyDMkMs1Y+x^_7?i7fL#{=!?z@jlZ*=dkcmYgSutLs+|O?JFcBboB_tzM?Oop)Gl;wt}3^$!y>4UoK7WG2WfFXXoNrf_;Ge>SZ;H zuiZ%Tq!ARbs>3bqPh+%)CeTIvrE@4l*$74MrC^F8Nv|Gb1w_Og3?rsQr4xyu@BFZf zXVQN9iv7->^$h@jAxJC(sE{tMEC+mvKrZ!%vbqG(Rp`-bE-<$G5(6C1^?Fu_XyBo| zs32;BghYFisMk~l$ulibKYO;YG}=sQC$K=vN$@?|Y;M{Pkbpn@9#d0R*zK{@fsuib z^^EAUd^y)K)kx2>cJxj}pZ3UnDuR-6!VqcGl_#=U7v<&oFg`I70E6UQ+8@{*v%Lp@ zgD0MfkUFHJ*}Y}XPdaQDJW!iX)t&dKN=m)ga9D$AnKSqGt{zenOVl->VXJR5_0Ymy z(_1@;pkO~E1`O~cxK$#r6|x2qe}9mx8Y?~ltue__#n@VsgK~ZzC}2EDGCy4Jet1y)~YseEo9V5H?UY=JMP4e0O z#AVR&gfI$7&YDIq9#yHD`Ev4pc<%W*!Ex$H|ALJ$cGm=I!F2a!pTqyy3C_`Y`WB%d&;@m zr2Hjb>EyVb*^oT~(ZM2D*F26ZA&{oa)XOqAej-x}wvQ4_p|L{sYiLH^Qhkh@TVRiM zLJlX@TN7kDJ&-r7#-Dx2+DJth>4kAI_2Fux0uq&$84inhz_wg_@y2*9c)G=cfs4P?`8doi}X2Z9ibmzm;_M0lUa{jC*L34!u z)Tq10G(Bvz!5M^_^N~b8{+QKnsNJ;C8`EX``zluk#U8WTp7MF_?7e^VG0xX?x zonHeB|1Tb|stBbn{H2a~-u1}8 zPAtZElx--t==rP{FiYvQ1LUv9_Xa552I+aKu+u4`4g#%~gUffsX2uKeh1#j}&egd# zt3^>i#uk%4Yd&&3s<#D8m&9Y8>kr{&c$qqNbbr!Q!x>@`kFe>5n{1S(fwZ4Q^ z2CRA}iaVNfJqy@a>)mc`+gSTm&tPpF-CXSx%ek&igb!@ods!6s~lmoHpXqPL)g#^v0)hC3eq>mT>)LxSugV`5{>1d%;`ksGg2F z-x|%4ctZN$P?)a65qxgZeb(PV1hWrQ$Oi9eOjnnxsQ9wo0318BZC)HZvkh@FZC;Om z_t?(nwdcWXL|(VVN;FP}yhp{<>5OvZW>z<_VE$k|?ORu?f;K4Zx8DgcZKB3~3z? zmNF-F$B+POI!)_dQ;R6rWa`Q=1qv0iwp^?8#uoS8dhGP|ign1}?3Lr<#=#s5ivA2C z)3MOD3HO(9cxQds=ST_!L{OnxHK&`0G;@!?R&HrCK?FM9svUJTCgh}|P8MfS$H+(z zn9`SsC0jxrx4Ii}|FpsSE(0XJYb_4pfZk1zmQEF=KXALo-EfdoX~-a13y~w26b)HH zX4>$i3$*T;eN$^wu3IHt4aR?6TwGk_^nY5^VYVkb8(l3o*Gve1J$%90M+*^pv-3R% zrF5qG+}-4iq6u1Szci=EHC*$Vn{RO&{J5S}aq|V#zR~MCOBWCh{fg8NinyD2z!qTj zasS>2k8mhq_a3VFt)An(BS6@`!L=-DR1~9o@9BNo*nfWTIdTU&j z%B#2>sMb^#i>=^J_xZfEcCjq6buT)LUMp953K*WMI~~%%7Ru%6s4tkq=RcCkWnb3{ zE6BBuyR78}kJP`^Af=B;Q}(?QxaE3+`e@AIEWLRBiptV6c$+RYMjV;W^HL7Is0%S3 zff1;{iy8DI3dshK(cd3vJ|wOcqz362{_S+Y#YYQuZ`I8OPm|DTn6N<0(^0|?`cYsx zwOzlX7shsx6NZ^sxee!Wr+5~aF`x9UVzMArNNv6|!!%={8RG8Z?Hrg%`{$il(5CL@ ztX!(&-VR1MAH*P1T(y{CKm_3V1d1Pw>}r#dqq0;S$F;?g)rxCWu1F&V>=Cb^&F3RV zV9$ES#|FQ${@3-qY5`uq3$^+r!nP-(OveGs-{N2hI5A@u{PkJZO3s5o_L$5$S`e}1 z%~GveV;s~Ag(<_#Adwf5JargmQ#Gp1b5ZP?{^&RAv)jsgc9T+scT zl=!f(n<+N*FGkHWI+L#PzL+gW+v()D1f$1dK60h_jn$q1e#QyeR1=N0X^f7g2d>tv zSlt1Z7AS0F;Uw7tNo2SQ9kv$zst9C_AP9;!}h3HVsy9Hv^ARpSaIy)o0tQsV)Z;ub~4habtda`dqzqk9_pJkNE`7C30~ zkE<4Yw4L#?c`e-1XM`#g=q6gq(hUJ;Z&{IO@F94XLbb=0alLFs#Fr)qG{v5im`cHM z3_1N^%4&})bmNtVtmov}Rm&AX?V$j=sj_VR(QY;Kx+v0;iM1iw!UkCA6?%eUC3&mW zcPd28=g^<8sU4q($B~`S+a7A)kEsU1pI_eZu9Wq;E0*^U8O38WKjzQRQ!_sg?@Kd3 zj?eFECHn9Fn~}MvZ=|TluPEV_hFV5XM62$(JGA{basI-D3jE9khItQDZ$*m7UXDI* zpyJc$CEV8;Ru4m7jy_eME4=u8AAh9Q>!e-ohX~Xs8OeK+0=+x{o69@{HflEIc0rm{ zQCZaaQF=YUJG>g$z-cQP~5`C?lC!tU6#a*3?xeR&>+QCMxvoRk5aE#T~k-F*ClK^OG<6j&#hK1s4 z>@hT0obr!4eB!zJV?Jt%syDxiFc^ewD+SHYN3f#cmgu-3-`6(V}QI_-bxe%`PCY0wLG#|Z9>(gb= z_K8rH1r0Lv`(!gkZ|!%3b86{EG?8@gu&hU^c!-<604cQUMy&05rQb^ORtVko@}{{w z@E%?hD#dUa$HHCJxlqZ{5n^h?nxV8(#agbw;gaZQj;hr`n;bQ$-ZU(yU7-jvWB{4) z67vi$Y&ouXt7yUoVF`EBFy?mNNYlhw$~UJFBC>w76$IrYqVpshe#0a`2SLN+E}k%~ zAR5h{?u(Xe1`BXt<=J6%m(2!KRDIKJi?QlYX|!wOsO=-#opl?gh7F#wUM6qQ#EEJd zG->F+{xyxyQhR_pS1NCF9~UVXnC~kQLD8zuBYu(Qf89MIIbzHxPvz#49{Pa0xi5Yv z4S2{+-eZYBXFg-+kx68fBFHp(53|75-x$ThNG>-4v%nUlPc9ef7|p^MOWq+!nwBDo zXvPAmq+VG+399$7`s==fC-`pIr#cUrTtx1}RWc|o2gpDwuzWfMWFXZy$dyXpt_ynA z6!9`qbc?1OoO%ksEk-3ZQ@z>jj{SVV9u$;(n(mT#s;+1#@Zhd5QR)i(k2J_dCc{ty zS8#$4dxmKW4U{qFPohyorEpJMKc9~=he`5krp`!rxfnE2tJ*>N!0JVjFA^T96I1G^ zvRjS2PYI3Rf-&5u_jL-T4Az=}RMb?G+-vz)(@2Pk}PU8lfMoJFu3nSiT?&c>35$A4X9r>kp_^qPu`zkSO7 z@#KQOz}NJ{B}OD5|?goT7U>kq{Dr^6JOh z>dY;2ud{_nw-+x}HjRkVPDxj{b=S4TQ&KJukXAg6Sc}6%5(PMk|8G+llv;$D1EDYZ zet|9F!DnS2?gjQqoPju;!-!lRw{~2jC!L~6x|*qeMtkfUE=Pn|Iwmz=YEF^3aKRE# zeKrj%o9Szq%nf)#&1ok*joEmRudez;4gj0Q|Gw|oQB)L8>7Ee(M6!ATYx5X?JDuRT zEwcGHKI8$-xcH;sHduRhJ#P*Q=S{V5zqE%S1&Il6yiBr>GJiH9eoSk!#sIoLNaEgN zGSne3iRy`K^n4v1mhNGGSeB8{qE*jz0wuo;L?up?#ZMLmJiR1w%mbNT50Hq^}|)8%-g*DbE^l9m$~YMsJ^B-j0^O zJrMTbm%5Sexg*(nt73)yHXw)XW{3W+J5&?RCsHtrcE8>bC4gL_Ddp!NfW*`_&hcE~ zyw8l^X*3}3Is0558FQ3eP3_s-*6)E){>Z8n^uoON(0{G+6JX1SeGVa&62~zcX`6a3 zPl2GoUi*|?p!$C8c)8)Tq`T5^z40;4hPiMz?W^2&oD;9c5-aSa(extt18lS_U(dvn z&cnzHfAIgBj16l%EjN$?cruv03BrRKS*X&8|@#(0#QFl=x~dmYx)~bgp<+SBdzIz)6f)^2AT-{ zIp^Zb1nsRd6Jg9c?!|Ijv?(x6W#lgbaSH)7QDbObYl|VF8!NzuSEgiYz~{s-AR45J z)RTFG6T(yvu?~Dyen`u)$G z$CPfNy!9nhz)4yAnwco>;G*mRFfNwc9lFrh3=QE+mE}&}JkV{T=;m%{$$~4NNBrpK z>>UpZ>iB51&f!8L0nyc=A0q3fck8(WKTog~xD_>wYh8gajYU4YW-WJuGljNLzBRy|k zm}pEJ+iKksWnLKaKH75$Kv)6Wphza?w#U=2@n#9xu;Tv-#}*;#!g`2sge+PPc3^F1>+ zsNcCdysPad?3zzDUUh7s_%F=Co@oW?d+9AmLo6yhrmhOApEL3B;I#YIK(*HibzE(M zb~VkJ15AY+cHYK2?MEF))lrxeo}nBeF;U%X1}Szf2!swbtWJd$4aTRVqdCK~sI3C* z+Z^f)#7lfFH{?H`kasgmU5+(MT^Hwgm0cM0arDTrH zLUnB><$61C6}#VscZ@n+0DW_gBh>gy;lDb01mhN`<~q1(e<0b!OhQJ2sN~Ln4!lOx zW;nAcWySl#Y_rzzuPz@(D&+-Ndr&(2P*^B;H^|OCfW4C6_oKy7#U}3adLXSl*yi!h z$0u^<;*!Tdf}TIygl#sL@B?ofxnQ!>9+Z^lDZ{D1(i>);D*)`xP5jt|(k?mt%`-6a z`5mZrptc~^@+YreFXnk5x62eD&Dsa)X`|Y85@eCB>X_v|YQasPb?HTrT5+xx!D{t| z)_ahs+uv~~VicNanH?XLJXD z7V}e^c^YQN?}?%BbAI39x&JD9cWz+LhOJHoeQrLC>D+#Zb3eXY2miL+??Uzr{CyjL za9v&S#zFt;mHq0ap8*%@;{zetBZQp2RUn?doi;jt1`0;+q6hPVcPW0Nc!?kfo=#nD z!7+C#o}N7}(8MNba6gvD(&u1lp@vrQyO|P&pm2Fz%3)7NCj8IKQKJA(_`2tARw zHpS(`u0r)5f#XIX$`+t)txtJwL>fFtuD#1S!Rsve$qx6WjOCBk3LWY(!5qFwy7|*4 zmZvyGbgF<+Z{Y!L@E&0_R(0TMqFT5JXo=2z)w&El=Pm_ZbJC(2*m_dTlO>7khohTw@JpVq557 zJmqMWFoA>GE46$6%#dv7Z|@r9XwZj1Le8^F{$q=DYn`}JIamINXCzB2Rlhe`I~cl5 zat0GiR7Zz+OHRoyv8J#V{EkgMMzRe)Q(};-)N431wmjUT?&7%t>oj=GOP84|&tUE_ zAkxu%LbicmN2?I7QUr|qK3Y;X3Bw#q2%h+}HlpC7e!Ti=m>jl>lU12>ke&N+KKIa` zXaauw!a8e)-U43pJhqE6QQT~;pl(HAJ-6Qkdka6c99`Dt&~l__BD%|#TV8>#?%S+q zC05H4tFi5A4JP!&k~>?)bj{zRR>I_ z72{=fNbsPRdURJRWI#B>gcG-oG~GazFa=yfJf~TCqZu)8K-4pW)U$#I92j zva}Y<1?XD%m5qqvSVKL5vTw|U8K8XHHTR}`#o1-(L}HccXJw0YQ2ZrV|Kl74?*~JQ zidQ9Nrqv6OsnUpvWp91YJ{t*a zOW4VIiWZ@W8?0F$t;u?Q*Jp^MZ^2xk*5-%C<<|w|!i#q7P{KPg%(Y>Gp#gtv#>vU^ zUZ^*NwqY)P?eHaD2S@D<9e6qWA;<9LI3rC#Dcl`-0xD` zD$)uldG9AaT8vWe&#qXhi#{7eD3VhX;ul5QF)3^t;!R(vtAPR)hcv6?t*z{i`DaNF ze|_Bd6fmcoSTHauzTcXewzE$tROPGA#;Q{Rz{mHI8N-6l*S71PtbFsVF|vzXc$}9I z?0nTWq`to;5Hr7H(plG%mXh|Qb$VA-Sa_iGN>4NlT;1Sk8}jp+B&rHrX$qEEtq4C9 z9|u3ctD2F}(sJn@ayggR*qvBwaNoMdLCY}{1cj}nZT5iPPt-VaW-%=#)hTr}!VFzv z;_*-VX8kdIz?`r}Sf;H7z6MWG!&%O4Zw_b5CBoqM>`g+{8bC24+cY!tC$?P!(buf> z=4p(M)I`O&&xYMEui>Z)Q$hGINm-hv@~LC~f^e+w;!Qy%u}T_pi?N%JEh1U6iLrd^ z9LylfE8b>g`+w7r@v^`5SHAemk03%^qeMEmQXPezYUbpan{K?pt$-RizXQs0Xi~ zd*?+pSe34LA+;%KY#DWljWhGAFxx^uer(H)nLc*&E6jGR0JesVq zKw}qNGVrbm^Hd|4IvGjqY+p08zSZt0oSsa_oe_Ev>;0Y|&=qsKZWvHsXXjp}?BV{$ zpU6}>pNY(Ocf%DQA|D4Oq&1GU@cwV-aF7D3FLGmLisZ^bwLG>lqYkA9{1T1w7W;Wvw&e7FIi&b(@l}O6YcjCA?D_0;utbT2UzkQgB zZe|8t4iX;dYGy5w?j!%?{&rQVdKiYK+iY@Ldpd2ohQfBPK3rUoQeib9%&;+tQenk` zn(?52xut9SbS!MYb{m?%i?U?uRNvJmW}DKO$$Q8oU_?~wz3?IJ*`WpEW`)UitccJ% z+N*4&)iOmfwB~D~niCACP!!s!P)xKlwL>+`Z=RrykhgLDaZrsZBXvR9Kj00l(Pcl}~a=DH*sJ{$mmT2jy6MJ@pdg zRmeDim#66%W4cn4a;KB2Wc6G{Gp^)R@$^|s*I zt>@plNl(bAA(S*M2bI-sZt0++LU^-XgsrRb_{QX)OajAKE2JW! zU-Tt6+X3&+gNccxanXJ4!#tvcg>N=t;Wlf#fGO`XY$EbAkzZRPIJX3-fh{VJdN_u- z(8EX~b;M}0*svdnfyIJy7>Sm#AZamKEx&3XcTZcw!lFs*C>^BESQOS{6Jq+MZUW+v zh8h7992>?&TcIPrFHwY9*#ctyQ7u+GKXuWXgMLOf{I(Fyuz9 zQZv9KsL~J8$A(=L))}YA9U`_9LNMb`w3@sO9QRanj=W{!azTylH<|~{=cauw@g1XZ z97z*_72VM3xvK0aR@# zJY~0D*KZ%`mPfvV3D1_avk=|6Or_db4I*epb)5@AHC9ZQuq}0evwL)M<$K<(JzQkA zLAPrz(xu7fRXW;ebmY*PU{c@d?K)hxsT0Qz6J_QBOUkjGDnaR139i17dmTRa;VFKK za}6SN-R)#htDm|PnD(x|^<;3s?SIGwX4Nv4rfyvHViJQU!mFx|pIX8VNiZbW0$l=` z0&kmtG&k*wAtVg%l z?1C%B#5D+}uc8t~@7)Kn%w=6-hgzie&74pJK`FXI!QuBGe@*MY6WILBS`0oRNYm}7 zwsyM+sC!4RCPA~qvzwYgu|OFb9u4oaY9A&di3NLXKRslHt7D5EF*Q`IM7b$hZ_yvz zyK~77EXw%DL z7N9A&kOe<rNKE@a3DZa<5iuEahA! z_Gzb;vz~_>l)!{+nf5fpc!n~va5@N{A)aiZm2~0XHS>zWfQI{8Q4d?oU$vP4`hVPw zeb+kHZLmtT)azSm)%v$(7=&SPjX` zF@)VXv;yKYevD_4?(rp>@fgaJ`~&m}fB|hq_e028c1CKCKL;uQrj<63?P#-O8lEas z+oW1W*u1KLWaOj5XkD$ZW8Y)dvh38r+c>)rN6l2!E_yEi6xr8%E%B6AeG_eR5!aC2 z(R-=!2y-180fD68BWxNVr9Jw~w6sm%BnZzBHSOe2U8x?!3uzZ)JE=jSfs0@08H$Sn z$Su6H%cLCM!m8Y1m`mk?d|}g9z#KGpU`zOm?SkvsgRbr78~m0TL0E+s(UE3p0xdV? zX!Wmwdagfh!kR-QLg%&O#@bV+rJfSMF!t+7cxRYe9*E$`xN&lQjeHs09XD2X+N!|n zqWt)3ewL{C&$<>ho??p{8Uwj&f|NB{41PgT`d${5w$O1|Yn;rTrPMFPyli4zrl z!mvaZh7S{aY7P@`e>2SRxaoKJH_|;@ics+(mX2({6`)aSCcv618+2ppWs+p)B7e&| zuH+&=qGt#nXgkG+AKlH{9B;ic2SB2JS#rG0EjoP(KJkC92*6zc=og?;;U*h9wRlN!cS}LoHFgnYF(#q%K z^69I5dxS!s|7DU2z?9AnfLh}kU)A2}DB&%So-^PssG?}ud0^Z1p=^V-x}QvCPuNb9 zt4*aL9~3j531e^e>(}-sBe%yhG^pU)4-pon3ltUs*m2kV zd|d?!7tzEyiyNzLUT(8s{VbZw-xh~9aR z>Ep^JTAEfV*=6%kZ`!80po;~2J@cl*I0)LbpZT&rborq+4Z^^=)Z&Ij7C~Yv#Dn!l zcCnaP*ThSm%Jin_B4nW^@C&QV{XBASVg^3ZsqG%?qD+O0m!Z2Y+K}$@%`L}jr=1l%o@=|!D6Bf~ z{&GuRn8R)J1A8jJ!HVeZckZIyp_ee1c`Pe!r%l7!JtG_0?E7Wq0b-VBC zkmQHm*&_cwe7dIfh4Xw-C5i9yWQd#Y7H0etm$&$g=z*$2EaPY}4_{c%d?qe{E#q~Q z17FnRIuo3*+i@aNcDv=67xa4F_FLO)fef+DALFg~Dvd_4*C+}FKZ!|Dh&4-Yr!g(* zy*L>CBJ)_j#%|r1VvU=TYstr3sxKNf-lfM__ipS3+_;>M0ztZuIHebP)B|Ag9w7b3 zM`b5v_1!CFjP`&d;&~8=i%6-3XVCz%Ums6`AEqDFP;~S(Mgav(F99;5Ug_eZgk$ zhjA_H@x!bK{-{*SK$@|JXSvB@L>4}HumH6Tlw2Bwk*JVZZsQ${ieWy`v z2OR$^%OFn>njV7icALhh6pkV!mI<7d-AQwKRpi^~UY;2C%nMj(xK#HN>-s8&uJALS zQjqtk#czUMb09x>qRU$M$h{>7?DX3CGW3zMvrQ?GGr|HP4jM80K~o5Mof1-Wv34Jx z)ef00>P9#ohZw{ELHJISxRJdZMVtbj{fbpNB>lNtb4=hiz(Qk^1aEEOCJJDaCy_tQ zd0(8D%L121Tps$KG!RPiG|Q!JCrizN@Fz&xlbFzG9yzlIm1^swN)m3up7k+>>N5RfY#Bkwkehobnh|TXZV^r($>RkyYOE5Vle0#*Yg7A#xLW^QEfd|8 zA;>=%>UUcTc=y`vvVb-65+At{O9lHH1ktU`u{2MCenqx<$?>)^;8F_uWgbMQ(2khX zB@EXOz)|GIqAO$OaTQd)F3sCmKZ`cKwdhv5RsW4F|YLZ`qdA9Oc= zLq+!G+3m$XF6)Kz+%C4EI;nLM@P0L~kKC%0`3cdO;v-Pm+OR_BB}p+3DJ>i%?6u8!h`ttq)@8s z90v?`L``iz>~y_PS-^toGPD_c_tY350j>cf62se|T%f4@oX4*nU3V7Jl+py_D{%&? zh!#@8^vFjT7$IL>&|+n=as4;Z!}-fTNH->K0`)JC2-61L&j%MkMn`S`S6dp(krDC`F&9Sn906y1HSXn1aa= zwGHVS`c1zf+<{F);Q6Y=$O_a3sj`_0)`P~1ug7y8_q(!M52nZ|rN_)Giv1N8&_sy{p560C}PFdE2q>4pH`HMpW zEh9~eE&8*}mJx+@FGe6EI%!s$y;w-Gd?Q*%1p*KyS2qsx^<@iCpESWp- zv{mubQSs1AhOEY1R6TH6BwXWnTIbaG{%7_TV+}RjWh>iXeK$n2gKCUJt}`;a96Ggw zv$p*g>c9+0>6~BsMJZ^w`y{ z$3YVC1kM^>NWQPo6fMo}9F-UJD+J*YSH84ddEsT=d6;l~+z-V&Gd=&NRxFB~Ef+{7 zbz(v9?9=m_ViChG-H*A_^9nQ{dU}6<%mur7e**XX9y14wRxQ7`LEBYP#9^Vcl4~Us z5X+9C!~N^0`z|YJXjx{xfosx98Ujq!*9U}GWfTf}U33pX_G_t!aJc;bPS3Mz6*?q@ zIO>&A1C-2TX7^+}+V%q;xkH|kwFmzVA%RqtScD8QVt^Yx1rJ%Si!UHv`hI({4D6qP za|;a_c~JoY(ie4^r~TG<-5%}Lt^88RG@|w6OHi%%oz1GW)Cq)i;3SM%qXE@?q%JO} z)H?KYymnY4b zouZ4DGN7?vpy4+C7Bl3web|mCz!{CGY+aR`!_`Ke^z-gLVIV#&T%Ew}K0ie8`Q43N z0j}p;*3mFwt;TMOIvx$%8@`;xJJcSY$N}nTmq{JV(WU+xTzx!RhyD@ErIg%IXGJCW)bN! zxlgsAzNzT3@f3b}`QZD4$PAF$ISlhXaf#6 z$U5H0k1Fqhz@)Y;yvy8G)3ON=bI^aWt_2J@roWuO=?FYj3aj@EbG}= zm!%`TgpE5tjWFxgonBp3$#Nu9MC+fPg+52vzo)T?&^cemlcip z9CuS6jO)SBy$gbNJ?^u)csrh++^TVw#MQDdYS9_#+<^01_3?oV=aEX02J!+E6UB2J z^W>yiZ}j5X2F0x~k~i_Z&;nw1F)Ca4baCexCO@^}9$U4xY67kBOB9l=&7|(&8;~Nj z`#y9^+%5nbQ|$IznlKb$T_gQG_C1UygGqU2gq!!@4tfgh(H*@3{NlbMEp~tV_0vmi zNBNw&0GR@3-SBpTpC%q$fQ}{K(szhJi?3=~yg!DHc5&)o{s zo|h3EonmT6r>ZMy+av1)hlnZJOdMm`uCo=htZOz^MbN(Q{HTXW)SD{}IT!=8V4vm7 z4uAuF*VD(O5x4BT8&cu%+r^1DYp*_G-rSO-qN`j>g|z;@VX{?);skmk*8@Z)W*X`B z-z7h%`Ig_NTGca+Wd_X-2Y{K&j@1Xkwbp;S?96_9TJ4+9wsa(J07C|i&%SE{Zs6P& zb4gpD{LSd`Y~i>6Z5QQ~618y|9#kTFZmQCUV1VdgJom>!^Y$nJaEC*&Zey#BR{8&6AWS;)y6R41u zoDxWnV*pk_slRFMeAo0C5pFy0XD}s8)%4?;YkOIk=8`7(vz(ERpXkirLu@g(Q7nI0 zBMv~jnmF3e|04BH^$JH~TJ4jzq_HCyS5^?F`2_vEGHj1bIEfYYaEIK`nVlIq`k#;ZMnzen=-!tFNVSPS>QZFZL>ZR-8cnKU=+sfXzAXZoK2K|p zmy5hE9XaU~WB@$}3CtA+n+$v5)H4|d#2Zr$B!F--f>YAA&pcRQ_LWzl*#;B zKOWz$-wz#ua`wjQTn!Q_GZ|X1gZa6?jdim{f5g~{w+N-^?=hSOO}0pzH?%E}@>Mgb zN{#FHn9cs${zKPjN59qa=K1OVN=;o93AnFB`2}>@s_ml0E;oD16$;T1Ri;-F4-r3T zQnVg-OcVsM{?T?>Z3ClA4B>*2CeFF5)U3P5*WptCXit$W{wT%675PcZ()-io6;bT* zr{vl4L!*I4kjtlq(v+`TE4(*J%1W6^ScZG;VQo2g%C#n=XEWvRHZy#{Li9}sT!o0j zL=uO_mR|4K@WYpl2duf7|DmD*@z+nuJLRL`4>fNm8LT<%l$3|tXQX%Q6YY!8kJyOs z^(JD)3GisAK;XEwp92*Fr(Yi4-6WhT<51b0#Ex-s2WZG=?*w5901Jj)2NfeffF6KQdE;-?RPKEGa6l7>mm{ z)7%TZEK3G$3#%Lt+pg4HUQP97Ea$TQ*mj0l`ieycN8a+u|IDT=&I>8z*X&*~NCK&d z6_mP3+3`D&41-}dV9qzWQPwGH>}?|-b~4;3bL(#KSZ#qC?HUBKV$P8(1?luWfR4Hg zty>#2s(gwThK!dU1N;8}pzx&Snq&l>J>`j9PP=qopirQ4FlWj7GK7IC`BwRahVSP? zPx4JuPk$QCcQzyFI&_;LJo;%e2ek6)+bxpI_UOf2HZ=7o)m5c6`b9)iUd!ps<1Yr% z73LsCXKo`*W@;&7yFomPkHi$MH_;ZG;-3d}T1-oD9;;IQ-`6eFpr~Zc0<*#V9xl&G&G)X>O^Q=Aub;&#D9Si;fJ4UXCKNF)KW!NC|2lLt+yY zVnqR(g8>y3#Atzh#!z8)|ZmKahw&agg$-3!S8!tuQSKB|#MkSX`4~{iW@k zwsDX@aau*IW{aUF)3ACp=lcndcbmYN0VL2&dvD6kMR37Fq6_WKt7b(IVvU^-n|kq_ zvD8Z>??Y!s3T2OQ!ECE)DnQk3Kicay~8>QX=G0O*3a`egae{oY_si z6Yq~~x7#lk^sivFxrUAJ=&?vMXl#BWr; z<)e!8@?vnIPiGL!DmU@;4n*#qge}n!i*s)U9EclKU1u4IG9G7KTbnQWTeUzzB6?Cm zO@p#TK+12<6Eq4TFzT=`ccjq^L1P4tXS3w5zZ8GRC2l3~(x)woI7(s2w9V3c*f^a& zd3go(?v!#Gsv9IHU-AecHAJQZo3JO2kPf|_O^(WCoQs*CL9tV*g(1VQ2cM4lN_=q{ ziA`|u;)z73?S{s=R|M9Qd8iT?9h%U1@8!FXJ2eea8z3N6C3XH~Fe*!5+441;e9fRI zf%iJbs}#{=3&Y*Nv}s+3pv09<%jG|)DKEaw$+@|vwJxHJEQj8L0cs|e@-);%vad12 zX+)8YlFc*PNHy9Mau!1oga1V9cVpYkr^PQFG3d*zQF9-L1;Sy}hf0PT8h}vT0*TFG zO)xyuaY0VV=-ts5KVEGF#qPy4tk!C35z`acu^tz0o;Tz>Ya-I=fGk@XErcV)3ekpD zH@9d*53K>Eeyw@1n!J8#ED||kAqUw+2)VFS$Kw4L0zSmXk0F|Iwr8OpRqf#;B#PL~ z;co(c8x?Zb;)%+zAiKF=KazS^u5GPNxM-9J+Cc%C;}47#GT*NhNc;qMwDyMROqg|T zyc8ggXYoHvc~cDcd^uh(7;jTTOtZ`t@0xH*qejI0lCAUhzp zc-Rc_#&j(_{?WGk&X)S3ze6qe^0#(}IXjD^Jm2nRcF$hji*arr_wu`dpY7YHqQAyV z0h8<(g1PJ$f(fA5E`*!^=jO*>kHQK-docRMW!(l*_}1B7A!r{14B-eA{jFyga{>(< zb3Kg=q8+#c@{4br>&FMa*oZD`65LH`Zz<0HW+$M8)6H9hR3DYP%g-*7cx(lM6Bt`` zp1MaDU>RE+7+IvQl`###yAE`b7aH>vmyRm24t){;uL|+3obZV+on2dUpanx{Yz{@! zx71oD>a&j>rQn{3|Ir)!C#@{0WZOzC;bf_D#U^oNHo)=PfJ8r23-74C++zg?u^a0K zJ^AR!M!>TJZrkwPf(HWX+Ts1%MQ9ZCmKlEGs!=);PNM+B94(DvQHuaWd4ec95=qO$ z1K=xRhTX6$N)4KhC9ne=YyEm9fEru^o=O2r36JLaS#%VTFNhc`pp8sDw-q&55Xi#Oj}~)fmwGV`w2bc^QBLkdEIr z#=?s_K_k9cw2bVvxr9-oh2+60B%m0jAlV&2?`;w5!?b&XOu>8Yz-S@1$2 zecve;HpuH%8jb`9%Y8r>DTdAhvj_8)~X+Qx#}MGB=2f5Hr>QQ!biv-t1>S}d+)LQ_%(oO>#*-3--> z$>!xXa9NhiLcX3FObE6bM9Rim;4q~%z{n>(b3wzDHdRUGT~U9#l(F$r!hAKlihL}V zSgQ&hW2~X5+!NjOXw7SXO$YMIP}VDIYrL&%GisIO$EP1VM)Qj5{N4Eiq=zNXTtE11 zeHVjAv=U*pPHso5D&bObJC3RnD0W!3x~c>LsW>YuR~QU(DIBtEk;rI*e(8uOBnKcZ z--H`?zse9}AC@erBLy3I|IGNsfF}k)tsa>K8fX6}$O3`HL zhqGvWN7+mYLV6t;&72V0UQ3ppfoC^rLq7j~&J@0H;)D<|C>x_TPXcyAr=xWpH*xEG zm35_i`jJQ-r^nvnH)~?CnKm=mVad`vHh?W|c~q=M-We%v5%IgFUq8(Rxd+N!BUWLZ zt5l?2?3vbtI$@7xcsaVnH#+31At2C!P(mUK2-K)0cS&NVfZJ_$MK#3403bKYxr4O{lCvW9;; zM!?yJjckIrDj+T0?qEBCU7xXBxhRSqgVbyUWvko@sKt*FJp-Tq3CwBEvO4#n0NP06 z`S)8}48@W7y#Y_ZAV{4*0D9|2kOE%-%*t0=#Ka;$okSpRmOT)gyPd-ex_YUU%@en` z=D}Jf-z+l5X<10Oo-mtj(bs=7x#_R>tg9!&S6w09NCt?+3u%Y7!Y6n~DMlcC?2-8X zBNCfden+_NQ!q+`Rsej!s*mRmB<*c)GFK$}%xfk(H#RHWR_HEH|AAx=sI|x@oT;_Q zQ6!a*KVz?hdt4v{$m?GomIy`mf$H;cVxKw?J3xr-Ycj~iTUdk1lU0#?U}5+yMps8{ zbn4p!*tsbs7jax3m&BI!f|oYt^0&k%%i~iJr+2{)|2ul_HvgswV(+qHq2sZ zkh2&nHmo$2#~3MF5GlJ?4Tom`scuOV+PpfrX-E;E2|cztdSr}eBg@ZDPJojX7c((- zu#xUvT^2JjHhN^-a2q`_Ml8TYP5|4?^qy|>yE8F1D$Plt@h_wSdG@xncJP0w$`xB0 zkY~rnYsaKH`zT`h)CaNNUAtmZ>_??JjmfeBSZv%yvN-=EGLr&>|9Rr(s`dX#klN+m zfBM68E#fNv|5r31&;EBr7WKh<-bbG5;KlIgbGooY91}T3c*?NDf-`TX0!Pjsz1R7j z%6pc4^4KuzBJJ7yOiuhlwU%rlHivGO%gWx4x-r{ zKy0cbSjgKi@s`r}Y-Q~^$lEV%j03yQ*vZ?Ytc(M@`VWl*yK30U+i_5~p4%G8XTx;n z4XaP?0BTN{{}J%oR7V7jW^$5X;Qi;1W^$_G{AcOn=s`G;N~8^(F=OgM{P$?yHqV%$ zc=23n?gg&)&>jffQlE=mx^4&vs~yXKi6~ zYvq}*$Z$n~=M_Od4kRTdqdl*mMaZn5C*r+2vvEN#DbUP|Jb}JE@IQUl$W$>mrc8CW zfq&1o!mGyLU(67KZ!UPxCPy_tQ@%q}l2MjSekh**U}87cqE4AAD=YJOA5MN)Or~gj zsZy!`tXLH9)BgQH75`p=->-Q$PGfh@9Z+%tM&H(%Tcc{}M+}cjhDwH=1Xs&C-$$40 z(`#4_Z65(^7NtA>OB`O=!F(E6R|?hBh#tkCb%p)M_~n{s;ZX68NRSqD?2`$e+m`vZEo^YhcK_{opxb!S5 zPF$RtCZVz?RJ|-UfMsrC{o3r0Nein&o>{he@sRaSU8wWs-)`-}_)X&MRx~qrpF9jE0t& z-|aznsXU1p6o#c#dSL_-TF2fk`zVfPAHDDsxhk$)YC2tKt(d2lnMuTIGAs(%#q}NP z@(D#tieVQ#9avAM6cnJDjhsW0UQbFQg2$@Y5*^A5A8K;|$JlWaOUNqH3s08M^`aOt z({$C-xb{plzU)SsFF@v*&ij33d?bPpYm5r1P0#CRCKF0X#cR=VQH*RshFr2$s9Vu~ zfSFCg1D^S)cNe^Ou({ayrc*4}$xt@82~N7>C;f=RVhA=Eg-*k=Vb(PyYBnd5!-5>D zS41-)Lkf_hgy&uWB#<6YMA00QSC5a%D7d+;J;U~-`1*#A6BbU{xsTAvlbqpyREvc$ zfKh+e$>r+&cBe}Vb68X_8^uV*9$(YHrf2!-8iH{&4|?bEa?zphGURn?MpZpRhv6-! zk|+;|(^Q;ass4yZjQDYXI(%Q=Ts^JRf66j=uBEtjoK{XO6jrAavwnp~c=pv%8CZu& zPK@Fjw6N)0k!cNtgdz55B|n$a>yVNJFQ0}rp<9`{J+P-P`docAOQp${}Ps zFT>qWX4Ee9D}Ql}Y-l?#(;O!=Y#;RF1DGQ#Y!~vg^J*96O{QcZd+$ z%1=AEjgvuajXgkQsB!CcRh{f;OY2}>$X8DNL<_1pUblh3_VFHA5jY_4jCXOxT!-Yf z<=Atb8w8CAih03N;f7Rht8jkyyO0$Gb|Vv?+tKIsyZ<;j$cn#(% z1eIWD2m~Mk$}#&{s3(B@!3&3lo64S@1&8}kBJHVM`kDT@W?zCslxp48nzsdA9lw>e zXv=ph*g9o`5?%~-C)${~O|fWm4|d1p_C{R#B+{BSeEoAO5{5=Xjl_Wt@wTMJVEpnN zNm+f1qvC@opZN>+v`r-0t|8b03hY*-9uF9xN z;kw8lIro7JrBkUv*wQ^U|4($s&Ld?#-PGS{4^Iayv9_1)KjG@E=>;0NMqwoXc4@cU zCK9U|zusbYfYLaLqqkY90X_NBLXsVF>XWAw^iLa<=$8};rce;fB^{Sn@GACNL53N6 ze?97oo^)5KfdKEv;M!RkKEeC&`_hkRYTkC;6J!rxCj-2bN{&?;`jJMiz+j@uQesNt z8cPtf=kH|7QqCs6TyhV@C+VAI8YUwnC&S-KM>35I8UYcix_jVUR04=H?cO`_w$6@O zzOvMQKKqUO&DMvUJW{W-S(9iUJw;}()%yot+BZJ!gw-DE)kapps+mz8l#MODisKT1 z1m(_BX7&@*rkSj85&u6xJj{KcFFc@u=iJHcVOqx_3u&H7i95MnbrJ#z7=5D1T>L*DaZ1}(_TrOE!BG(zA7zpe;r+ppLN|{3 zwb|C?>Bdwu_=Z&oNMf}j1hJ&d_Dw$9Yjo~2*vZB+I%8q6J#$vRV3`aLOLI_~=*IIt z*X-B`f!A)fycUEgBxCH_@c17YcWdPKn4tWEw;IIW{`^CZWd!pTACX1br(&UMac#xB zhg+`b>J=y13T1(KnCD$E?mgN+^v=1a;mkvf0rgJVP)5$q!7*|6*G>>Sn}y^qCPAXg zh6VY5P>12L)=hKflT%6*JS$Q?b+F8O)u<*!XQYlp5qdP%QMNChgcQ1X4fM?L;n{%RL-~JX6zDa3wcKPgC%k{a#0?fJ!Kc%a^RB zekRd3li$1^5{9AA5RfYW+Qkt%9u8jvDd8}fL9U&OWLL|x%2$s@Em_>RRh>p%}sF_};nJfS||+NdMM$R~QABS97LBsWbpfQj%OU z^I~UU=A|gqsdmejOo5w_qOood&IPhpD5#D$L|`HDPKBS{+KS&odCEUT+gaBVr50L+ zr0@%^iavYI1bPv8I%}D%lg(KySfT!_T>-CF(l^1H+;5@s9P;1J7C{e1dCIJg;1t4m zT-DXXuBpr8TOD8v1uArUS`rN)3CTjzqVTl64E8z)$?GU-zbQiFUheAN94GZ{}cY(cH1uG4%K(;+eetPzA6k?K@Y z`z%J&`Eamb;sq|Gsg;_53RsRT{xC|hLCmEo57vksn1W-26PiY^KSxYUFTp~xi2Y|) zXBQJyKx`zYRj*!n&oe-$yepCv46AXRNPYgJnuCdmhNKIw_U>M3JHwp2C8f$J#=-(M zd#gC8;Q{f9(c0!m?7IGO9GB#nC;OCkd|tMkw}-*sfm`(v??#H140bWXli0ivZotWa z_#S*4nBPvMek8~#Zby-q@v7UaDFb1ejFiNIC(K${ZJV4z2EI*gw$xr>Sj&7osu57Fu&Y;B*Ze`q{^wr6IJ+eBN|B1?}HA?$0&jAlnH^S(?c zOzX=wdybVe@{LHAu~0|%;LIIaamGC&>FLYP2)gI_^7ge_sSdTtl+FbGZV00%+CY`5 ze`EJhiNkERe4qHkx7b_k{+LgPsCtK_wnoh$Vs@=K^m6r>MFPV-Opid$xax~I;qhrk$f3bb{+ z&v_0r2`NFbi~c3Y2L0mFu~Jkmtuof1Y9%PoKr@qBa3K~D8P!aVmx>D&kcL(i8>EQj=Rc7UAwm7wNcw# zJ!}|HGX1`e3WNCpu0icn@AE5>U9HQ^-iVz9-IReSmh7!<0&sLddb8zvq44AdQY#X&z*go)5eRQ z4>5fb>Q$dnNrMTljU78!TQM%V4kAksx1G00VTM;r_ zC2d3<=Hsu2!e;XOP7OkTlSMi7iG}rV%!CKjuRmh8H<$f$FhcTA;}bCAF34f*t0KhH z-GZt{(QRz>P^QRN$<1FVzMUXbWC7FkM|dt1y9^$l?wWFOCugt$D1nZ|l&>;)P~x)h zd?GB3`Ldm+03XLoNO)xGDOAZH5&LCHLIYzSWc?g>9s~-|QMd!}m8O2J)(Y3aAhF8X ze|HPj3P{}4m`|K!SUZ@(d)dtOCQu$<@a$Mn*L4&U8%fI8gGb2u^dc{FRk4 z{FML+wvTdgg5|R!Y#-Gc_W$gav;P8>ZT}_t39Ark{g7)9y_g>q~xP&gEMV~2J zf)&)lnHRPs?|94TiQa#)2y+%|y1VQnW93r(Dn8v<{0rKgf)^thOoeNWE(yD|;0ZFh z9&?(9-aUoUwUH@1waNr!o%G+=jY~uUSnq6x^~RROtYq=MNQ=?M?2Qf)tBTMylE}jO zy{TP(q2a##bY7Y#u6_qP6&rkqk8^=d*vjVU#BrwuyZU#AmEFTAD$Yo9AisSIH?FRbm6Px@9~nX~cm+8lr+B^uVqOA>_sfT52ly=?B?t z7SXoC+hI>O`qUYo_G^vT3 zR}4H>NCB#pIgID5C$vY{d7B*sbjJFLl*EQzqAXLUl` zY>`6ZZ=?ic1Xy0F#y2HFKJiMPvfaN8H?L7j9Y)@9fCoqaIfwQ>q!pfOnX414=p|hI zp|SQZKZkTXhq0|}uLmmrPyvVA-r<0gcwqivpLYZTIu`rnImVU0oc$V@mlV3;w_3}X-vyv|I$hp*vtXohFu%x(B%Y^I%!oNpWz+Y28v6y~w>c2@L2pkc|M4TG9S z2rNUI#E-;QnMTnb#nM{uU>_mo2o_9W7gw^PLI=m z4r^%*=6B#sga5R4(1s4& zW}%=B#GWcBMe+#|E%Z&+N4ErU2CT@*LGkYi!kJ^sfyFAHYnAmJ2V#Qnl$+Ultz2XT z)e?fKf^iaL6X~ClEn@K-ah4fzo@0TIxiFw}OmUirwK6#Hi@N50cD#KfYa^IXWJm^C zX3-f9tog-vj!6N|Nzn%CNBh2(9xeMJ_TSm55DS(1v@`O=Q=RS{2BwVh+TFbl?>ygk z@!EfNC)Kp32J&EgODh_BJIR8TL-h;et@@6@qnn^fVEyrcDVKB=0sETYVQG8?utXtonl&>87G)EwS=Xm}JSbjT@qjg+LVmnO8K0>p%XCAXya&TguSYh3|v^r8t~s%(7B)pw{qL*UvSPPK8V>KwIQ+Xc(Ob-$7&wBfuyjxVco zy-#NYamrw&U?~Bg-J$4Xw)!(47ePmc=!uPD>bptbsVQs1jB0HVs9xoqJzcG9#zR&q z*YjfIcxG;eDXYghLo0z(G{pnZ+#gn8*t0>G{NL5R~B1s7=R)$95Wm3vyliZVWX zKUSoj5`jiW*B2)PRhLK8D3AT^88w^8;M$E|`#J`F=SC;>ra#SlT(l6d#gr+EtSUGh zbLC^)SMg+MT8n>?^Vdke>Bc*ng$836DM+E!C$iQD=>~y-&2bnJdNjQ1`gX<$x*o9hKuWHH?6QxfhwI6M;$}qDb9SA6RCydw zrP?pcD6x=#g3!TFD9j*osXvF&;U&d0H9;G zd%wFWjP2D;d(}>9yUjdW8=H8dT_AG#UbGs)G`1W&7gz2ldCn}SsA24)2-o!reZ{Fc zto#xlSUNboJ!MYgMzs89=eMpx9yl~il*2v^p_JG_gVFpaWiP4M$I)fiKGXLh~|Xd zYc~V?s#?*sSzU?32)B=OjTA_?4Du$IwX>gvB9@Ja(NB@d)xQk+$BSpbJo?Fd_?5F6 zn*bBH>z|d(7k9U)1abEmt9NR$PNu*{5LTJ0B(^a5%-)s?yFJ3w9%!H@CEh*Cx|l^& znRH!WQGZRmL5gzf8Ewx4KRJx&NCH1*Ch_b@`G767af^11a4kY!K3*uuK=SGxjAk1S zXX_76KPHVdkX!ic^YobKmxJr)tTvSQqE#^85C9e-8jC?Dl=U zoIH#$JiV>vX*P7S{C+&2Q9K-^NfM8Z{rv@O)<9}bWhQ=*Rjb3D6wKYh^aW)h>)!`b zfrdf`;B#WhuYsx_b0Do!z8Gj-&9o~BB>Gr7MC?W^tTCFvK6X9!S6waD1FI+?(w3%~~Q;%-Twf|Oe%Ml{tIaGtS^Ke&RS_% zN89m#utZzZPxNe462An*QOuIsLTRpOV>Q6fHhFa?4@VgW9--5x)- z{E)6QtI5^d)X8;O>A!kJwEjz?bVD``S^7;E6&__t(m z!rPJ0&Y`<^cHsA@{^6G1EpO%}98jOV8MA2}kEdpg8&7FsW^( ziXZ;4TNya;@^2QM%(9iEk=&|QX{b0G8q8A6Pz*Zhz_*@AI@I=Dzu#wrc8w6pt|0bX z9};5ifeFdbT1m!Rt6UH1lb9%8CXiBrsb!gGQ$5tKM(iGatj?rid+#EN@^Um-@Dm)V zll5Z-gmd}?3MRyhW{?5b!e|_>jCuXz!N|U6|8Wyq`@~tCn z)cY4e#Zn>TD4~g+SGTh3Nc#lC`CF4lZ{LS?ngMqnd~N$_D&m9XH#PpP^Gz0Uf~6lk zaI#XpOz))Lk^D(ccBHUG%{kbAL=)!uy(b{15Nl{>93lPTTXbYTW`C5I^S8I>XY6_N z@wAR!zi3q#NyD;fi9KfM&}S^$A%spKtXUK|@&am5jo^Fc7c`-=M1J&zY-9-kR@k)556o#Uk1)N*R5y3Ov! zh;KL!1!e{ZyntErd?>z`)kz;Ki4&exTnO0S+JK0R$)R59`qH@XrwHo6th8awJ@W&w zDRslsk#WYen^K5bj&#@}9}qr*n`cEsl0@q6AogV+FmskVJvKF%ynYw`avfTtd*6tx zem6=zltqly_b?|R64dupX8!v*io66oMYpd35oVY64FYed({mzAFnt!|vL9b+djGpo z07lW#bN`0SM_gYKh)YYlW^`ho!(8?3!$gviE0k$kTh9#*l64Nl05w>-RS-!sXhL)) zylf-e8YQ61`@gZFNrF2qS24m;#7k9a7FKGCQzxPt{@ zqp1dEz0B>FU_K?&XmgEa-zq%TE9ZG>eI2IrLl}WMOzXGC(o|jL>>0<=5w~7VQ)nNV z@>`4vp@aTGP`@=$Wr2|OOxKoXmOk@RztGRdJ8b6XrA|E|Ga2bd&S{#Cb@Tz=9f`~# zQr`#>k;!c(PKtS2*q|Zrxumtnu+~2<>8}7gt=zkDQLG*;MI!amJP>Zwx>F9sn~OwA zVudXExju_pyF1@wF9S!a7-~-1lr9Mx3!eoAVY5zj#*_;eHQ0B6#dJ?;tC%?q$wuyO zKc4=?*1$#HF6t*~ObycLW+khw$uS0o{5XWAlAIu&NV;G3rEKrBj7ti7Rh0rda}Q!5P$j zxIZF#4Q|t?3x0|UyMxP@0J(1hFwkUP%fRK^oLV=aJPeCgVh#R?f+r;P1i%g8|+ zFYSD{FyliQ(rhbu_?(r+eTUZ+BNT&mH*Gh1j4&NO8DlJ)?3P@@K3m7nFp!Yy!Vs;@ zB_<&J@qwM>9_edVp3Tb|q4}^P4}ZNxDAtrz7ae=?#~Gzhj9qfcrP0U0*PtHGrH|qR zX2+XevXwpUmMzbyUkrt;ls0SYxbng6v2w8p#m5@e6jrORcF$j+*S6)hv#woFOWkaJ zw(F z4G-s3c<&afjE342%C%sVFN2P&Pefv8ffGdv<6+76B?4cFlZo9_3)(U;i(bXl0^XUK zN&*z4nCNpPIC&1&*0rarRM`;R)!h|OVFmP{!)_2J& zc`1=8ex4pBxoCmMEIX!n#SF~f4-$^Q3KQN*8*n?*kLX+kiyXBZ@LT={yt3lN1Ghrvp^o>tr0Dp}i()Qsyp>j_M!VQ!EqVpa~1d z$+SVJW+8?lQ)Es22P-Y!$=3|`bq*xkfPS72ordn-z7gz}#*C5@tSl=+Rr6m- zcDhB$&o}&hs!4TYlaAfIrm1;#BT6=)VN!lq#W*FWVRH7x$v7oP8>i|@5U1+Oxus!J z-Vb4nn#VXLr-og99jofLCCcm7tM@gX>4g9JN(`2GOc9_RmghKt@2BA^a>fhie zEtDt3k^0>L&F=}ThMNe{bqti0tR9$;;@C~*DfvCQbxe0ZT@5n~=I=55t{upgx`COl zcgiu*{ZL^~kC04FXF8z&s|vhjEe<8ZOR-WAnow+=fE1iEIMJA=_Js%ZU=~PIT>c%S z8n~B|VIicRo~b~X4RsUHUK=S3=>FnN1QWav7f(fjluU+E_cZ!B&|&7`o!#&dQy^S^ zrq;j-if*c43{j<=puAr@l|zuD=X>+b_)cF(lHoB)4J6N{iReCutb|WjC`y+0;z^`{ zq8a-#*x&9RT6jH_kIrZAVTxN3P?^fr1b&(vTJdXU?xsW-lmZ>lJJ)X|hmbeDrhG{Z zju{-+zO_k2tU=#9iJ%o|e;lZq7}pS$?j}sbm+{&?MvJ~)HtKhsz&@@gZfwBEpA4<- zV*aLjZt)UIA+GO_FPmx_Ol~Q%N65ltc$!423%A4?P)1NQ?#zIeKCzGo&IR``LrLNX z$9W7&78U%xZW)Xj8e+WyN}<#YR4Wk^e#6cw0tc$74*VWj-~9~nj{@v`TTH53SYU!RzQBT27|;v zU<;qwU9{|yOjP)3RV(;>`@&+_74Y2b>-sYCcZNwwH@`0z7+)>teFDws^(Mu7YT2;(G*_f4Cd=4plo-f%h8n}g3 ze4N*GiE%O_Ue!|lhoqVjRUGzD6Z1NsHz(6FWfj{4?-97uRRLHD7f?Ji>y7N+hoxSy zu$5x1YsG~F`!@is`*^`m1GdVe?B#B9*aq*h*2Oo0=~;h&(jW6R*Agm29O(~kR%|d= zfwZ=M%WajcHsQJevS|1`{F`ImtO1zZL$Uf=L96L&jZ2~y?f0O*pm!d|y{!epe6SX5S25asMuHi1N`%ugmCM^|kI zHS<~MRz7-->Q1&V3k+#fZi2h|88JFC%QAY63U4XBV2PEg8U{OYf@BNYrgSe)?KDBI z!}db~J&oyncTZWyw(L$WZ&tb-j0T^+kR;^@?|VRa859U|^=d#+1RYzhZwx!f0m1q; z1|7JUUEKf!Zzm*GEdVCjw!Q1vgxqYHkT#C6nSr3 zTA>h22dTae45aAQ?d`^u&hC!dW39%oG!pwMF;NA&G4mL-M})Z^6q~hkFVoXpe=tPhl zljwez_q6*b$BFpoY%->U-*a58YICCl_s`%1rgw$He=AR0 z%&>5IH^LOE%(U%;=ChX43;L$8>};a8$Snz(rjX-mGwy$$!t;q689#%P94~Dty76jxHf-v9&{Ap-e zM)NR>%uJVf#uXf_&}i@2GPU!aE~bc!RH0K9#{H7q4o zI>tx9{*ig?6T+90y5T1B*xltaaEwT!8z&8_K!*-n$Xg%48T5pQW#qLO_8V(0tLU~; zq^EZ65Sp-q?Mzl%^94Ti)O$AuNs`<`_G{jjv@wF}QSisi8_MLb7KR z@)(g45m{%k@R&*(aW}%mR)&H9lOJcBVGZu7jc*3DvwdoW9v<~LCsIyqW5)%Mu5$v;HU7mDRVrhGUa1eD$>W5h13kQ__ z(QYe6$S6)OK*5 za2w2NSh{A<@Lc=}b@Gs4U&Z%8ibvBs48Ks*)OR->sdXIu+iw{j9lqz}yM9+N8NxRe z9PR-#6^M82@7E&2OgR1f-QAaW3+eHr=xaB9{!J#cp~g+rKBjpnV?rMP;37cKg`8U+X0Gb*FxRZ*^3R{ZL~DF`B$tp_etO5<|f$mse1c2 zEg@u3t*QOL*1gcPu&T5m+@`w~(*#qWgpwJxP`e|s)B@aG29B$Oz#rV)e38lW%=97w zB+^4pvO<}qGZL>g-&&=DyUx_4Eehc(H+e(ehbNV{ThUi%BhL*0@b+s8`U;g&x*1+r zHI6Nl5acG+D8(5rV+fv~yIkj1DT*_b-^_9e)1RAWGjadWw+Ako5Em@DHPLyjWhrh5 z%C3fDG|6%^^LZ9)XX&+Avky8Gf7vJWpj>r(cpWd2C!?n>iQa{#fc8)!u-VW7@?X9e`qN@|7P`d zaAoE+mG&svh|n+}M$dEomIeQOsB-h6$tYg{E@ej81kZmW5W4Iv z+eS9nb9G@8Mx7i7+dqj5^a#-^cJ0gvYoOG=W{Pt`2yk;G^RpMIF3-yK4{+Z=U^2GR zez62D+f7<4%XZYw)P|Ysl`1Im^D1 zLfS!PWr94lB)z`Dj2$9K(=EgzbcZ)69anM*CA7m^ncD4-DUlfuKdywE#mDdnaqMB{ zqvK+pb5n5BKyyqOd9$gFrIz_9FK8M5&!{1MaR6a%q)=0c8Pu?`}9;$9%}IVLa&1tt|Wj9%mFbsR^3Gs-xr--(DyjFR2euv!W?0jKh| zts+`x5()pK`MDU-*VeJuDZ_4q7ENCi36)(C#GOLrSDR8Q+d}St|UONxf?+ex{KmPCI|IQUdLCsh4noan~*0lMGl_l}zW7OHz{y4m! z96Za#<2)t8+ecv9&i5sGc8I3qJRx5nA~S8}`~7`#ilDTe-`&KgN1B84l(KV*Fx<-b zW46>aLgY9rdHXy>4z4NU42I>pgi*F#wThJVzW@ISb*Jb|oT%$FPRF**C+OI=^~AQ* zQOCAz+qP{x9ox2N&Hw#w=ABt<)~bs-b@r~huCssCFuZzoPq7-rLiA%}8LT`76%T~Q zB3GRm8+;q;P_h0o&+pyMl|PHW{vv~=&Na4T&FXjcw7(wj4=?KxgQH-?W+a&Y+ChXb z-9K+9*2+v>FQNBX?Plg{$GED?Y**5NeZf;#t4p+?rW+LmHBylCJN^G9i5F!fP^gEP z4tKrD-0asfE#=G$!^@po&T6*2ESxez*YXYNVEL3|Z)kFuthaq)z0tzRABhtsq0u%RJP;P7)>Ez%7<6Bm!X6 zSkWo%k%V~Vc_T4?X5tM?_KN^ExQwV zeccIz-!asQOWaAs~_t_%IFhL?&$UT0c`4fzRP7ydBrlMs@^o=z&D~A$MR?NEIL9oj`R+`+<{#ga zUze8B3zuz6@X2!70g%VJ=50^U{%P3?e|Enz2+4t?y5<}YHB*(qE}J8 zT{3Z>qamFmXzHQzQ&Kf!9E#Mf{yL*lBpDY}6hvWSB^?QL;@S{AV`_0N)>|BCs||md zO4_lR^PzvK-VRx@RC?GCc%)}6)#AD((B`kb%=_wOln15eLlG)Alet%KSt}XZNsJ|z z7xYbYIIZ9d7>h=01sApLN+U4HMS^}+6FnHR>XMAc^zDr3>7t@Aanj(6$6=+AJ>zMp zJ2P?|fN#KqR(Z0BE@x_8nD9~BhA26%BgwOx0>MP=o$_?Tz)+lA$KJVHC2zT^cz zLwuyKn2)jc`^1H~oVsqord$x{G)~Enm!oqS;4~G%b;sanCQenrTLk0i2{OEVa~+sY zLo!Tc+}nQATO=}gfqzqf3p{U!^4Q@`Vtvwbo^#QacUIU?Z9RB=zRM9cUllbIY-mkN zxw2oc3dNVvMrnX#5>u$Jd~Ch!>VZILl7pla4W-i@XSJz{>FwVPe|>Z|P_I#=GLt~| z8Xi*6!L@H>P`732tTwIioMtQ^=rx}Hi#3x%oL?bZJETYW>%BV*Jy=Gu|_yovNIT!Rz+mf+{>X=Vq4+16pa**pOla?YCy69qu*DQ;BDEKetn4dCekp-S)`1^ zHtbo!FykDTx{mfK#rLU6W>T-P!&NBK){Z=hPv9kt+iLc`2Okj-hKZ#3Xix7{C${@E zdhBDohcXJEI98kv@;A`CqQ6Ls)=&L`Rb{a>bJR5y7?Q12sTzxSj+z(m={vFElNW)q zS`Sc-qQR8{axlWJA_y?-b_WRe(>8n`wZfbi%V;JxE>e}ActYinqw$|Do%-A;ohWKl z(!0)`s8l(jltEnTXOO8y*O3|AYnX{jDXX)d9E*n9gGTgN#r>gF65#oA<|eC}nHkSU zFJ*&23FsQVba-jVwh1#R$5iHPHtlE8!{XUB7G>8E7Hf2z8YHRZzyBvXM6Q%fcaIK4 zy=N81pm_y&HW)UQah$?qDNp$tCCd_Qh^LjR;G>{;#`(%Mg^UhOerLlJ)E&F(ht~bN zI~Pvt^7VaUtMs%HQ)y6w%3Eo$qNo(~SLrIbO^sHaPq}UdR^U$V?3Xj-{Y$*$OiWxC zq;Ya|ZzldB!7B?9N*$g#YNiDyqJ4EFy29_ zR~)6-$f9-9FqzdXdO3MaueE(dB)734Y(a0UdRk+6?v(${d?S!?hA@n$fk(hv%Qp~8 zx@Mcklc9X08jt&e{4jIoULq74hdUm=)|ZjIi3>1q+7+nJVl76etO7?&jiP0TR+kht z+PKNSZ{U3tE-Qxcxng3iL$aicl07LU2dt0w4h+H%5!N=J$qS?Yt#GlIvwtI>-A`hH z6Lv`e`wfkPXPF}82=|+6Ehntwc^AJq$303S-+M^<--iCvO-W_MsaW<5-+#Wg;3Q+D)0aI* zfkJLhsDDqj1ooJTxW~r`#NXti{TJ`$ITocEFQD~i%VkaF#pk4+cQX6%Rr@tSdI@W< z^UkT>85TDSRy7V30%EKXphl=603X~O-B7in-x>{3L7NuqP@>FfJiRqp;VOS5!?Jn<~ z5Fv@tFst+jJZqx&%K92hxyO*?W*I%5h0Eg@AeTn9#dK=dZhewSKpZ+oiWB<6ptShN z_2s*TGPAAZT=QXc{46jD%Ybm0dxBDfoGv?kjzj!s|9$&R`=S9??F)4|Eal3hLSH&< z!+1wz3XietDV46r(W)tL~SjX z(4$U_I$pxKY=UR~f+kx|wQFu6y|n6|%Z8D8meUA$v`WFWt?3+s;O{$!k8UNzLT_*t zS$I$gIs>ZOViR}P#dZNDU!{*rORsJ%VtQ&UMkL2+`F|0dwl<--Fx)_BHn3q$$isoH z9enx+2GK$Lqi;jpaD#wRwvr`Y#c&)jWzN}i>KX9kX`y$zQmoaDc{pGPuzrz{nn(L# zo>b-WL?P?^0ZoaL&K78xlv1Gj#Ke?kw{ec>-{_-Owde`;d_FC-sx6E?KP}{bFN@)? z_-}izD?zk8FVVc(`CobeKZ5tDDrpI#WM2YrRq)k$371DD0&i6bBDI%ydrCQcZ`EES zBDF`%i_`Xn4m+)p6%6b(cQY?x$K~-?A;(Be0`Depz-GX-6LU`P$|`FbGrGJ4SKGM~ z&BMyrjpoZdEYk?KGPd4S)sF+^m+Et-#e^?B%eN?+Pr*{EMbC@e1I>xul=BYDbub8F2n z7V*c)VC(!rS6|Ew8im={`^eL>k)d=KwUURq+8y2QTQ_Qqlwij{ba2?95gccOuO!%3 z5oxP*;JV9vaMQS2(P?8q&IpEklD%=PC{hSzs&H9l2stWBgt77F@(M!?Lxq1$kQ}J0 zdkuNE2<+JNy}S^iBgLF>9VRiK=!qIB_}~3h zZ=By_OWj!osq)W4l;g~2-x51Fsicm@uY`%SlQ8_FXW!y!H*NHP(rTHN9NplF4)(r> zEqjK+>$!-=N?))5D_UbkIfjGf%0I(r*wM@xgjKG&@IG#?0La&0(E))%EG8U*H4fXFg*OhX z$|$go5PFd9AUM{9(|m*fcK{T?;9@?*?e0ff_;jbD99P=FD9kcuH=B3=A+6%%O{zC< z2uRy)a_%tt94}u(5kwLpS_a+aHD$cShlv1&&Y=|)HN(fvd#nV@ z;C(6%AARO$l1_SF1z^HM0O-F7&g!U0LPmMLpMDbPX@8xpv>!>`q|31N^3o8ZzD3`~ z!1I{=MG?LGV-qECIW-U8mYcxrT9|-^o}aDy3Yo)H52eg3ShJH@ci1NDFVZIBPH4jQ z%g5{Od6^BuGzH_3*H{+p6VcwUZ(FIL1b;g~CyanUNIRTP*+GqkGb#FLuH#TXhhew( z&kOGKgLGsX16gCXgy&-(@SeYZ3BI8faqu(!Q;wwb@4K8rp)l`7@SmmF97Oc}CRxDk6yJ zlSInIPzJwwqSwUPd=T+a$)W70NzJel+)FtMpeqkH_-7_Z{XFOBQQ4AaOV&N%dieVo zk%ewPYgPV)sd;yTeZfFHCqW=S_D~^j=+j!mN(u=>=pxP4C9>3%$JyED~12 zKdmm7t+L6%c;rfG!sQf;?-s%+V)Tgiw|{n>-w;hy+z`yl{*!)j#ZKDv{BV!`PV;gW zxo~~sOH*Cmx$7QXf=rl0pGG4>Y?~30&!6G1ob2>Yo#8 zP?lYDvCGnwzeB;m$D*{;)0JUqq&U3yF1DGvA2~pEqpu@ITSWE5Uo5JB;)N|xbfZsE zY|gc?fS_vGZrC{wPG9Zr#4$e3I5J{)R;Qb1#Cn2+?QUm0gumry(LhzEW4-*u~4)pxhEyvRxP3x#MeSY9`!tAoI#)b12ZgPCXxDU z#G+zqyb-{$7yzO0#p$~msntO-tETnDvXBX_l>%vQB~$|#V`6Z7$}%x!A=7S2US=Q_ zrP4MKe1omc@t^4(Jh4e^zSK$?Mx1?QUnZ&l*<={TpIOOSj9(?lUBc7sW!c}KMbAHB zjJvhH(uiSM8-+P!nO$<6Y-DEl@8kz$TnBHa$W3!kfl0s)|Ef`D-~Bq+`b`7$vOQd* z(onLye*gVaQNKmC2B4!x`dI6*`@wP$AP;Mb@%_c%D3_^O`d5RrZ>ym$mShhngwfIx zd&m!lOnxr-CWu(bLdh<`2-$h}NPZ45d+Dr?9z3-#2HF`A=grt7!(q^AYt=!&7N8?c z{yUx&b+p-JSD$Ufilw#R%^%|BboUNgBszXp_7su^&1%s* z$MSo|dsGrHb_a&z4?y8|@ekGgfikLI|XqM z8TWiF5oNLuOO-zxVhvK1(@$x;u314t1Wsh~vD5NaM@wVYV0Am%{5k(5n99dQ{q-nn z-XQ2U(eXAR;B=%eGSv3D-geB1d|oVlzE2t(iE?Isu5NmHKI;6Cdij1H-l#rCZ)Rp^ zXMA2aE^2r5yWe>PejWoYS~y-;OB;#SDtoC=xdeV5dLc4hkgPRdioT)DCgC3DJl$jj z0!l<%8&KGw1_%8K*KwZfLKTzs58Eiq@g|41@mJ4kO|mmMN4k@(oZm&%D~^uyxZ3_Y zh#jV^YW-a(2>KfrBxlK|J9r6@yvt+5ba7Ra(Z-;#nXMD&Rq@ZXp9G5_9VATOn3Ghz z!>oZ8O2CO&zBC@uu)}?Ovs*@o6W4*cVx+u5A+w~b3=cH38dM3#SExHAJ!$2&x7eei z`zP8J%s6d#h8UzymKT$-R2dC-Z7wiVC4~=zz48yYUY;R>r$fcxtFhJu5_65vkd)3^ z^riT^=AE}D3=sjy0bHd+({n-pRw`&Iwbcm9@n)J>1umb?5ZFQ!bC$i2+x^UIAeP6y zk0vjHSLzP{=~9+w2lp?K0nR?FItJ^HEze=-ih}&gu34Cn*=jM3Xk( zYwFq;K=kSN2p7{IUXqQ%;~BlOWCNlLy-LhO)WIzCEuMYj3+@Xp+#D4)Rl^~6NB`Ko0sv`ceA4Q>r+497tu{(> z%W|FtjqWrL&mg`=p-??WL>Hdv6T7b+6|VrswTy0M9DrbecdzD8;DYq5-_s7%8Q6=qv}Rh8yvB8jF6FZNnC~yw7o;yr zF(9PwXyA0TW~Kgdl?zQQg7j2wnV5sH|9C5+i|>H|)$EL+t%?}_SDn&*q|@3xBSX#i zpD&`-pEkF*_j?3)GHWv#TW(7-r{}FW*&@ahYz@g~kWP%D(@34|@;o|luJ9aWOu^q{ z7E<{pMxy;2kuEPpL4h>0xrfR0_5JLx{VZZoo^K86oO8$0{hPHNjN0m|g1i=MR;$?x zDxrT+(qrlxvzGVHe!cA~FQ0m84a z3cQp9Eb)FPJ3+;JTeO|gmfWXT|Fyuo{auMA58GW?X_T1jR;>abpNw9N;p^Bhwq|!%^R*zIqcL ziB+#C#y|f`xgH9bH+HcZDC2ATOSORX#$0Hcs!+}{r1bFe`V2^T4!&geX)~M@-s|2$L*59=oO@ax9^w3-CxFez19{{>6!q%3JTa;B6Mt<33^KAvyCCGTmt7HxS$n<1aptizwvdum!64hzcy>NLZu;_jxx;zZfj zM;M9i5dtZzX)$e#a~5KcD^6iy0Tr*tU=O z+ca9MO1P>Ac_4AB=pF?OI|?gKe`O-b{f#rrnvSVk2*V{bZtUfyQ@g}t$cRnL`aKQr z_ytQykxoF8rWqwVUCl56dVqc_$}G$H>eZ~1WNhY;v4_gD!kWSKf}mR&6X-iiPH`@$l6U-N?WNpE#Mds z+7eiV;W0D6V!N5AB3E~jvucs4G^bnqNEFC|XXW{(t#LSe*=1C`k!&uBKChd{8?wxR zHz=gfgE@>IBuc%b{qt>;n-Q7C@k&L_msd0@G3cm+5+QLqZkK1ie&M5ZKl>;^x7Ew% zpRQW{eWO@g3<}NG2db2S4&?{q(4N|~Jta8|JEg(Oa}VXxp`tN$T&&dL%C;fY=zj+& z0v&^;Gx67nPqe>WlOQg7f@Iz9^1gxYQK=s+V}{Rp8*clAc^(7cvnp*n#Aosf`05n! zxwtSYmg&UZfb?t|qt1>)3vKSkyALG203B_L#cB4XeL{a%u+2>mVy;swZH#?~yaD@u zFJ0wWV|01yCslEz2+-vi4T&W>%i^%RhgID3@mn38%GzgUL^j_qW8mJ6egHp6yoYO~ z)zeM&OJecCeF*<*BOz2-=Egun%tGLlZ$15}tY%Do^Q@~>~F)=@;W=qPY4 z=b9UvdIF;avz8ocAD&STcUpqlFD!LfMK`lHo$CTF%S@Gl?bfKF5<}h!Bp&5ANG;V>o!r*q9(g z>_a89Z4NqmB9}&cThW9Qw-+g}wxr9nd6U?hpi%WSEESGu7wEd0J%Mwi(Xo*66*B5{u3B{z1d9iUTPm!*q{DUz9KQqL^t*~up;Khz~?leT$5IAyb@$($9 zgMeG~1kf#Xjv!rN(;9_eq(Kjr^?RuUNf>PgOCXWioi#HE+unypq=I^S&mZyHssBsy z(EaOhULHG30-9ns+(j6-cmiC9^-17f(xA8A*I>@6l>&gR7`a+x*0Klswz`t?=Wg`} z=5i1+T`Q=IPU8RmTPPLaU?A4Cr>qEE+p3BW`DUy;(`s_o zfFzbFAg=xm^1lRv%1iteT*s;gr>X`tf_dZQ?E20aHP;w5*XJ7^;QC#SaWX+$jGAjm zjd61J9VTJrC4S}QrS2##&nPWV{)L9=iH7N@&i|ALFZh+0L1Euk{e`h=uFti`$=Sxq z+1(c!rc@4J??ai!$=UzM@xPY&ZjOPD%8Uf_wQ`=Y?5%I+opj2y6@~^`2FZZ-O8|xz zdR-4v_%e$i8BVU}FAE|PYh_1JnHnQlKQ;*Wr)RoWD&$!85?La2<^Vga|{w&1tRk@R`#!A0pZ&`OpV^1EOI-Ll4t||!Dl|_~1 zc$d~SLaN#=wXPZ*S*BfQNVBv^AIxh@j*-Z&mLcte_b6m{KiUkt6by5^hJEB5NKgfq zAccGjAH@i@IiEt{nh!);$eDpI8)=~notW@Du6{flgfb53$BV+xmHxP6sluvKgqB5> zsT>?f7$llCpG2P(h(HH8K{SHj0?{6*+w<^r106T-`1%!^{q}LN4kMJ600BHEZ=L93 z{XPJQygtS=v%7QWg@HkY*Y!?M6a%d&>|sFALD0= zkdK9SW}6YYSH{2&ID0BV-tX|t1Wkjj489L9?);#%!o*{|9jJs!sIizx_NhGiw-<}$ zH&>O#H>WV&iNJmAy6Bby_DtgMzdnBE91l3vt1#$*EK_j~8UJYTg-b&^g`_9=Ik);x zCadW)0G{J^?pAm0GiWOl$_|j!l9R$_9ON7VD?(YREpGq7)teu+*I>TZ78(wXqVMi8 zx(AP;o_O-|S9uN@1T@rS$nm)+px>AIO+veaX0g-wU;-nWaPVtl-9}EC0|?$hj__0h zw7B`u{eN2}B~>ZwSOUKZrkK#hBb^)B*0*|jXqKrrxL`NY;g8+?P?Op~L4LE^D(H`I z@U%R5V-IP6pobp6YHUaR2|(%KRYkupo171mQyaONBxc5R~MvvxEf&UOeNR@a_~_t!K0k8KhG)I+<$cGJ8MrB&2(b+w)_p zLG@e)p|$sc2HkZgv}Nk>efC_-YMTRJnVwxg{~ld6L_>?Jy2<9OAo+rFajK^We^6fb z+?JbBtyRj)=2zSFALiD~or8{e#LN?DZ2z63L^Q{czd1$?YUIfLe>&t|^u6iH{Pem_ z+7qmF2`SR%G(?BfuHRVsF?|j3JSK?=qOh7Gj+K`abc}b#8p>hNTDuaACPn3l;axSE zdik$jW(e91p3fybDim0Toi|YL7)!k^amA0u>NZQx!vtw8 z!gnIuwl9l2*HwX^Ou@2apaWv^79PJ5S-V{mCHveG9k&u`8!$Y6DpC&%LmkiqD41rO zdhD?b5_4&nN{E?TbYM97C%%P5}LBtSt2cAQFcv`NqYksq_1WFn( z!84OW(iG9BvUcM96O_I#bYpBEgV9xCP>ob~;6m=i)8?$|6 zvwnFq!PV9d-P2RJ?2~(9k)QP&Q=4I%IU^QY5(z3hf^YUmTB}!pB47)HC&94gDU4>I z(z#S0iHtfEcOFVzk*fM=T~lls%1DvIO%$3;q(iByrpHsA+Dm#yI{o8`?S&BAh&6(m z26XCI<5~XniS1fAaT7HM`z=IJJNv00ueKL6I*K2Ek9j1ckG;4ZqXwkwt&5y>i&ca* zu=iSi1!P)2X=3|k4?9@VBp{Bf(+o1}Mqm{55j8<#x~(goz3oi;7@Mi20bw!|Y;!pA zmDR^xcQT@89d4~Zp6$o=C*1XdP9%={Q#Y4*j0P%p08CyH4vbsyjZTm0Q3>lFF+kkO zVa-wV&I81r*ruV~kWA}L?|l_vqnT?;b6A$NG{-?>W@V%SA6g>R3gP(-j~y%q8Krb2 z`BU#S9!ttPDZ%4+O+=UJGQj+=LKFngs;or}T1Ac^`%dCcp8%Pz&(HIbb{VF5ofmE~ zM@4ycJ8{$DPQCOW8Z$o?ZZd?~AKuHqQBjdiHCL}|I)QWDjiz+*p}SQeFRD!I>|=}& z9e6FjqGWoUzCvg+8?Yq0ijOuW`3f2EaUsZKY>jWTTpOE(_&6@CRC60VyHy(T z>4!(MpmGScB24r*I}#j63#Kjpnhq|^Ek@SObB21LG&f(gUzWe z8T8i0dQc%`izo}|*6r;B*gsH|O(F?28vm@yp{dHG_p;Af$IoQzG`GHQ;;GTX|IEXZ z4EscD8o|4^v5n&T`U{v8FY){Yg4F!O1cjdb39D{3iUWxNtpsQ#!CXwIyUF{=@Mq06 z=s35ho5>M{7+~b7v*7tW?Ln|rb5Uav$zzfo)}8wB>t82?r6g}!Aug722^)@s*jC3X z#uu};#+L8J)=&i2p1EFt!d$eNyUbq%Dj9Dmj|AS>q|U!WTnb?Ek9HSxd4Uwc8jug$ zELX-}+QOl`;01m|WW@5zC|+$ZpmH8qbj84&*QO=NCF5A^0JbziCXQ;@n3t3>DXkQ8 z&kp7(Fe>S0>~Qjz6IL;*T@j;Ez}Q^t%%4);%?B$8G?+<#=!1+R6mQMcg(n7p$6m{Z zs@{gi8Y>p)mB$w$Czs(I;Up*qk4rd>-*KbaKdVg;d`Q=;i_x2?$D0tyuJPaEdUTa+ z7@t&TZ93B{4GX?G<+3Se@`BsEFCl5ybDk{1Dmc*)q}frB<ckj!B=I)Bx zhuS$UaH*u)fYR0wKjsI+rhbZA%G0$Bx_49a-lbSf76%?sSI;Ue!f@42wH%`30Bc`A zKA1L&!iJu3?`qRXn?^&E=^%em2&zotrX{cH{U-LTRQBf0g$(m1YVr9+`X)!sJ3zHt zy;FQ#+AJbpe86p%t&FC%iVRJq z2R83&me}k#fp!Q5(a_C;2<*oq1ympptl&9tMPsXd|(kd3D znpKs6PvZ0!`63c>1-gDDaI+%;yd-w_qB^f&*5m2i7|6;s*;{e=oUAc!MLpKriB(&G zKDHSg)Xaf2?(y!fUA_`02P z;x1>8?N1s7$lpc?4duy-*XmwRXN ziw{@$rxQo`XMMdGWqW>OVNKWcYbFET9K-M`r}j{>sy=)*A*6E+9L~*u>+ON-`xlKx z9lQ3<^S2o`-zi$rF^Q|yW8a!v*InNT?`o%UY6pwOkC&uChDrlzwdY9^GmlbRfArZ9 zeLKM9P0)&=C=d~TA7hP7jyFaO5(KYiesc*;QrR-GCgfZ-sNEW<$d9?Y**qW&(Fvy1 z4mr#AuTw>4*}y z0=dU2&)-_BOI3(ji0H9M0I@alD{V8)#Vk|InK^_SK_ zDCUBNy8HyDd}*!KD%9EWAO4TqPKAdDf!yq`Tjy_k-=D4ZP)M)WtGC=)fpr0&zdk;$ z&x1R9zF*(x%R6sxr9`5Q-Tl4%*Ml5#4ZCByaQeG%G;M=LR;CZFm~FAXb7g*o^p70pzKf2zL{tfvk3ZmQYQ zB3fGiiMJOgm(lM))Q>5hM2rDyt&Pc^)*dx(f7kM~otY^JV*Z8`V^tyK1n|w%r{5Z^oa&i9{>e5zzR(`h{^Z9ejTF^HhA1i46n!u(eW)~4CKWTO`EM(D4Z`o zFCMFWpIt9re~yAKm}~2yZ-yxT^KI2(Wg`z~OD5&oqH!Euizej}r(=s7-oJV;&Al|^ z+iAvo`JDO4^33JfKB<1ueA}sf5Xil`xdGv*Fi~l&jHrGc4a<1YL(1>TOmgrvFQIKY zhJ_6Tm3|J1=)$fctsAKFs-fTPQ9A4CW~J0Rm`Dy#EXX+ODiXQt$Y*#f>Ae#iGxRKR ztfoc|k!QiaB`5Zn0g@Z-F!rAw?BbNWRoz#_)nzV$RjL)E?~0GH^H zwLia%8Z>7m4ZTn}JqRdVqs+T^9+Pyd_jJ45xl(H5YwmR*G6JVvdo6tUX)EyfE&5)U zq7)qmU0KkSMQtpi)No)`;Lk>c&@LNEmBEJWYraJTc3Rakd zCOv7{hoZF{AGVm(;Ym3R%1`;TXjT5|!#3}k!mGvw{FSM>Jnd|_l_Xr(ToBZ>Jei+Yh^O-`+6E(!cema|}_w zeA5P7h_McyRlMI6(%_kM+e#{nzkJz0Q!ptRLSP3J{fJ0t)jdm(qBd=dgo%lVZkzCvgOvuGaaRK=n$D;3}p#1x7`t|}GacE_M%a7nAGZ2V0si$Pru zlv0f_MG!rfR%(o`_xicG-ZCMJ)Isr+4P|wgiJt4Ic}0y%8%)Pb-?Z!GoUK~fPyQ`i ztRA~WBB=wa)3Xy+PP!)Ttvg;K3j=HutmFfPSxH+*GmdEq*bgAwUWP1lX_XHQHV3bL zDIdc*0$LG2BxYmlb%f1-R_HbR#Tz#L_n{)X`oMY?gPxe_qj&mIAtMXF*K0wUN1*wC z-NUQbbOmbM)-vjM_#uB?s;@HH9Kv`uM3t3fPA1`lt@#kg^j3~+*PDLKXa)Rkh{ww-;6xU$! ze;&5N&rONQKHCw_J)M&FX2{EvyI%~1c*8-iC0-`oZ3qP`B~ATnd`Ea!fy^vXxK6-m z;2*3Q-O145KdQ5t0Kh>Y+pAn zs;T~aD{r4NqX!TA$5M~D@*g6Grt8_Mz%pzRu{%iQI)ClaY2>ohqUYyA!s%w-o^J`f6y|;j&@v`s?*2F}&P_vFNZhuj zFpQh^6>QQfM#7SZN+n+hcX6I>Amg1R$SCf&b{4B&oJVVRzs7G5!ZPZ>dev-(s2fK! zd%T&p4@F4Nj@Gt5+Kz2-oA)%IL*`Et9&zw9EBlyJI0M1$3AEQ?J}=`D8zlD4BfJO2 z9#A6liqcY^5;}Y!kqpPfVw;ul6l=$$^$wtMV`U(HE(at@$W`XYm!Zqb#-J17AD8XZ zCpjOFPu#+`Z;+!iyCMyJceh-G`u4YQ6iWQl+ju%#wdKK==vCbyYtj5aX+;ebXz6oD zXbY;_m`Tk#+22~^FyYCuGA_s|acD<>C9X@KE+Mm~HAyw-QDb}A_t<6%QJ7`>;2Ys} zH#dI(!Zn+LvSL<6aI&b{BOhjd&_~n@si2C}G3L@@skV$Cmz_+8$|*XU``W7Ff(HxB z_iu1C5Zc$6VDG2)ZDR-{7W-qIw+842TIOVSd|LGfp zPB)61;TYvXao<8*0PI;4Lf9{nOE8obCEQnAz6~ z48+;_;-H!!zy`y-TiJelEV#ij;$rdMTN@=O1Jqygza)In1$c`QIK+0hgI6VCmHGTJ zPn2^nibN>cMJv1@)#aZl)4w`R18(bQR{2G-v7j6V@Q3SKe1L3Br$_ST@AwK7u$`&P z@}^H|*6kMvLp=R(r@`dyb&s11j(sBjmXK#qbKd=di@%J`UdpnbC@3cxN4>-pDf`mf zVgH5MB+?5*E_H^cPx;q7^P@@IX;6R|_ZDaadB*PkJggp=Y#8DEChs1^?wIpWA+$-{ zjJlyO`T=h}&>L@j$b;F<0VOn(l50HhibYySqjW-i3avlV0JZAPSSy!iZF>vm#Y7R= zO?#r~6WkB!^DYG31}J4D$^n8`^pVs`R&?`BAzdt>Ry$J2fP=;pr z-vk|x5aA?bum2a)G>-dQ>)B@3EHbb`qP*OY} z*IuFdoN@P@FQFpsm!YBuXBl6rE|Q0wmfMeaQ^T6vt1C7aA1Q-QzbSokWw@w&9W zz=t}UDPE4Q0ctJ&YmTKbA7Q;f{&z=CS>CNGCZKcU(g|GrY|-An0S#D ztv&D+7{>!(5DSt5P!j#SAEUVT6(Aaj)$p5Z;@j=8plLSzR&6kmkX-7{ng%dy7mD-4 zc1$TqBbE^u2~ZVz+9*{<+U^%qnsR(0Vdce#0ZPLG^&AKA5Js`$bQ3PxUlW^=!CbWn z!fnoN3*v7N5ql7a>Ixo5;CdYHleG7RaE_@S!w$Vo~Hrkj?X5dV%ZgN4&Op|-z zG+dvD9PG5#ZaN1G`zc3WWi3m$Yx`S1q@x6qt(^&m`(O;fE`SLcJ5V%ZE6?Im?bGu{ z?95S2Uf!mfUtH`|vdME5TScLJ!FP>t_K?9rY?$nG7CzjGlG*pvz>9)T z#pV)|S!+2h86NdkTgVQ+FAGOykK#lj}$!@A+OGt{>0jEPLx3keTIy&v< z%lUMVhnTvkgRA!?%D_(&Q3@rFRzi%wG#EWf$NPm8{|baD2N`=ud>Bu?HK@HsteI_x zKugomtgl4|YKBk%6as9MKzjc#qgfE++Q2ttYe+PRA_A;Ow>YPsdNW2E4(m?p`(oMm zE7g|i*JkhRkIBH%UM*2H%;o5It6Lppm@uAB_7@T0m;y2Peg4qL9}JE#(yfiIeq}ZS zlv=@-YT~R_OA9NGz}Plictw&^5Y2GnkuzJ!VN;?50LM<&oKzHNpfHS|;uzP)1qIxo-Lu7VbGXp zY_(a|fgroA@b+X5Fn^)Bg;5SDG`1*ZGMpqE*~u)ihsWJ{3RH|6E>wfLa5b*TWTEZxU}IJH|I76O{W$Rij&EG z0}(O?FOFwNT!=g={BU-Ye_8JC;DfQ(r5+1X!^&r8K98?3pUX6jz-G*`jj#o}%lmAg zC`F~wT9mVE-3sClvY7U0q)BOPegH{8w!fm7m3exOVOZAgTuX_X2YMdxLiU3*QpJk& zo7JhTbBEZHaAQ-WEhqfG!`st_A<#Cw)jg@?JBMN2`uGA4 zDE%YpM)pq-0{6ZtTGhhGLApw$LluR&(tpy6{Kxy(7ron4{POEN*#{Z9ZtG4E)$ZpU z%1uw#2&9-<%7{0guRW3Jbpy_6Bs(m6@S-Af61x_wKVn~F}F=Src%(t8w6PVnrq@RcY}1a|iidWs1?jg1p8H>lDXj3s50i>Eer zzEk5-$Mb4*{;4Y?SNFzf-rs#zUXv0gjMu7#9uxAuX^8$}K_C##go`9th2_(BjZqwV}&t9NI-Zg-J-?$KHk9w=q26Wh~H^JkWK-?B#!vQ(KQ^`DH&<)9M`-FeQ zRMB9-f)&FMvu^pVLMsF{RWCZny7z~uaLAy-kW`55H~i#@l+3`RlY#;ChQB}B9$l2O zlrly5{NvU@n&IQ8Abtvs%I&hnRI4w*($z_#Qb!09TI+#gRW~c2{s@kL|FKCv;-MZ3 z%X@tGf`RK{#7u!ycAlauL38DN$+cJeQUj?OoyHM%W&;y&=kcMrN)iD|T%>_6xW*N3 zzS{H1$;5k=D>XnXKjv`5M?($TfL!}>N*6r09Kk%$s-;^{K*^=wUZA>y!BbxEiB`on zdy~&w*$O}R3q}k=S@eQuU0jDuI#y@>{-`l$E?ZWO#hP!JFgN>*z;iT0_{USgt`bw^ zYa}B_F}4r)9#xU#j(d_$Gi3z5m+$6f1GoSk#u9m!!D(1hz_~ky0#s`lD6HR^Vg+4W ztQb&<{eyTW!p*zB5Qdc%*zcL1r;7+Ek@K%g0Lo#((^O5X zOA%Eow$l{INRo#_F73g_O0PK%-G%5N_g(62fDVf)bH^4%`xLG^4IsYac#x}-hEdh? zxSNRI6iq5*wz*u~Az<(<+x7=g&F}9q{#9w+csKdgUS}>)!2(QdSlmT{CS5`bde`OL zs^w2nwb1>hn~1zC^Z03)EO7A4Au}H2yVAyZ>A(1@*2my!q|23HQVH zEUX;7{c#ymAJJXTDsldfsD3M#Jd=3wSt3S_tg*Cs{!N*11%&=qt51UAsGOfL_^NF5 z&?TXyuY&9g{ISt!9%L0F<1P9XuFMD+b^`WG|6i=!Q(86*qd?(o+qP}nwr$(CZQJIX zU0=5SWZSi`e-3*H>tr3?tiLr~I|A|>Qnkbw*DfOR&73Cw2 z6dE+LWYVALvm)u!OV$!JdE{y=SBA?NSO&>5mOr$8kDFj6W{omBWhgp-JO|$jwP|Yu zDzugImwt`^ZAp>0OOuhRuT!XNTp2>bd-Oi){K+(oa#$PvQ{R@LZIv(4o~Z4L4}FYJ zm8-gZWt!coedLI?>mt#!S?%PnQT6J-*eAYnkzs1@Tbi}@3a^>G*~g_;UmKvHaZvrj ztruAzxGlYsFBqy;ze;iMQLyikLhlhPN!IZURlh-zu3YVRRliM1PXp+9MWa<_l-(F6 zmMiZt0v5DL4AIbbK~(8Ce$F6^oKYmswn~?HeCX0nMRT=Se{#~mvY>TwtPQ(TLLd>( zsfZcpnZ@%&-leey?;@mpu)p-OnH7k($LfQnqOW9AG`}%Y%D2N1wX;_itqWi?9dFbv zcS4WqX)0v0ajdvE{7ISZfF5Qe>{rX|YE)|4I8>WEeFwEnUE=L*@GU$mZa}jjq!wDO z3o!M$u#b#AcG@4fLS5F z8uq8CsVS9Cg`#Lli)XUsN-QbbrsYfdx6hHe)%VU3M!AT+#sLo-tFx6k77^J#Y#sS@qJ;U~ZlU8cM!{K6G{sFQ#MN^}?d^aJFOnDdGMM(eG+q}t7b@q? zGMwU#Oq78~$o`-;U}{%oAEm2I+96RLy}u>6=3x;ETJwJo<`c59B5ZriY`*FUrnY|l zASFnvA~Yr9&+O<8o~S$PqQfCBcms%ht>oNYb_kkRT(TU*)Wuq_CqAf^7c5lK( zh%W0)Lufb>mHov(5-1;rHqg`0p~Q0pW=0(_7aPD}$OK(lq+vkQQFA zTOzX`m8Te^ueE_<(1!h0y^kBm;G#leDq27I4X?99dSQaqiv5MitVEV>tFLxCh|_)A ziU63q^w@(ndMXONE=H8mlb@X7Ro zbFSB{vVcy;;&={PaS=8c-6_pcB>F}GVjd{c)#xpI;O<-Yt9c^IpYsGvYJ)t~7^AnMH6f5uB^1&GBA1R-1Shs5EBR;paC|9i_ug zF2rmNl8vf=u{GCi+k|jAd`CR~d%EPICQT4A7S=RtulU>mp~0@-)Y15yXHB14njx3M zIdhO9Ni$r=&&kh^Dq*4^VA48lb8xpY-veUWbzVYU2btRqr3>|<5%Vlq;Ii5=jz9Rb z)6xYQ3-6rZ@w=#2(l4X;9G{1xzl^tA!<$nYhny~l+l7fgm`8LzG3E8{&Y?$BNPz+WKJoqB$H5-@-8rfRFriu$U5aCAM+H^ zT}H{Iq@Wj+(Oo7PKuMuOMWISP)sv*XQs_)t^jR)7f8ds%c*}|3Cz~D&m2&Io8W|fih<8iGHDGN*T zUsYM+ztpXCPz>|)BdjcgWmQf|-CQ6kUPSXt-z(@JRDjhWTU7!tRDhKbYgGylR)FP@ zCMhJsh`bwaH6%ezK1&(NC4!BzA%{_?84x^4w+YgPRl{0f7Ah+-@;QoVHL>~(ibg}?ewS-l^^5$xz|5Lg2?88rI`^Yset8@ zC6OWiQUzP!5n1DXQU!lAaV>FOD}zkUXRw!LNXUVR@&G|mJ0KJwgro)BD?$q9!1w_o zh>ep+g%ed;H&Tu1D+aN=@5P|gq|t`05Ob)aFMkeVD?~8mgXm=<>2f&Yz-HW34`V1y z_2j~-Wx}a02NB3Uz0eA*2wkYMPe+7M)Jg79)623WyusL(B}P(~MklW3{^bHQlcs!mz0 zJt{>YA$Mi-e5|ZL?~Q7#iG)&t zgLv)*tk)(DvA!vtpYWkRU4af1^@_9MUQs1qa-p764}`q&k_sas-?I^JLql!O7bptWt(AD8dr$V5co8#BpmDkQ=UeNvD(0Kc`l=j<+N_E`{?0Naq+9#f^}2^9NAd&!%rClNaJ`j?HB;q|9W(Lyvp&WNKf z%NPYkpBHZu*^Q=8YrK;u_i`BKUzW2+lDbW~y~`K6&Xc$wnF_TVo1(!nWJT?mUu>2D zMZ_Rfh1>wHtcwT8b9*nf7Eo$w;L1RSJV1eb-x-vyRX0oJu2J6s{~&^R^FUFGhNjD; zZJopOA+HO;bb##-4fmsNpy_VNig{!Yas<7=EyLM?6tGlfuLr`Dc_^GwPOXq~g>+(J zgr(UqDatogHaH)_(MdKEyWg&AKSSqhG;H5@nw2%(j$NtSauJ4|VOx;*BXy04XU&;> z+QS-qSu_wWD%NvqA5sU5FXkUDR<*0E0@u1LexWcS$DJ?`=R;**H76+9^<31a|Jp2U zi#&Pjfl}k&z1Hy-lCT{e$XfKx%{+$9-2|@qc2Ho@ssl5;A}VFVGBekz%YPU zO^DD+FObqwi(C1)tUwAg_;pyxuwr*>J7$o{d2BpzTz76HOYA^-Q|4WRY&F(JC~R#2 zA5V`o?Z$Q4hKraMg4-d*L@(GPzg2kQh0XNXfW`rP*$=YT%FsK|#agwsKPP}pX!EG- z)~)B)-WiNu8!l+W7C>;Aksj=a3|@+d(2%n#94|UCX?i>k0qpU<#Jh_PFq&D7gLkPN zeqbePbbZ%I=;Zg-o!c9#tYC0Z7~ecTNYBfqg{zi9lqkUq13H2$-(Yy+sq$l1&}SXg zCh7=QPw3E1&wg*in+Y%FXNb>HI3JZy_-gVHt6e(0>C5Y$KSO6e?b+Ilrlf>x+h-FW z;?i#?nW23p(2X-tC)Pq3Rk_6Y(b{j0J}pUYc~}p_)bp&1V^Pkm1A8t-B??823KVyn z^sErwT)B`wz7}9?Fd^0PU@gQ$duV+MA?t($er!2Bdu49abc6Yph$SEwExah0W!{kRyuuoNPV%HB5Wp*&H zJ>kA|WQ+OJyK=DOd>N%VnzEx-mieNot!k$0wY>3x50tF0&MhN&hz9n9;;oyjoMJO0 zyIX{*3+Xv3@t zn`EUhYGlqOBpCa!%~F2u^I4RAb}eO}Ly!Ixvxs*|on@Fh9v_P*^g8yhGxfRG%QTxs zFZG`RbB&05kc3Y{qOllX3BYu$rz1#ArL($!pvKZnwiO=p6i@gmh(H7t8QNr>NBEo_ z6zGVJ4GHP;CWlYmcO+XUK6hOLR~=9`lB6i7j%XV5syIwGd`ua|PfFnTpDD2nWJ-8$ zWkc!x!&87Jg#%$9f}?J3vVhNU8D?}FC7iHL#2g>}DJb|_SydXKEmF%bB{>Zul!Y?l zbxi6dtW`?z2Q2E)$qZK+f+EK1msxmTm2?m%gaV`~@wuAR8TAi87Y-W+k6`Tq1rFg0 z=inYnYnQ9SIQNGz3Xv9biNQ}6X|H??b+VS@$bNDG8H}W?r>6kqdiZ<)y4|Ld@9$p|+XV(~$ zyNx&HL&VdB`k`|C^>x`Sb(9U>!AoLxM>1F&tlKRjVbIFv{Yh5fG-a}z%)EyuDU}mJ zACgE|saXuZQBELG5Z1(!n|GAh#rHzLzRr5vdk+&$MLy<}vypp=?@j4puPC<&_<|tQ{*1#PGIPF{-fU6;vUVqAW&HZ z1+h0s(iDF8R@-fUB&Lr+KsG0nbu>XGp0FO9#PzPPtf1bA(-Rg^{eoxO=X_k1%wT`1 zlET#g0=~~>SaO2(2>rPV+HI3v8QzFg(hQToQ0j z`&s(rnwZ*y-$N9*xfyqDWLHOcYP@CD5uaU>0bOSXZXIFSO;&ugd7WYT@)7baSeMdt z4W_%C(>Wx*W~rX5@}ibp34kNObv*N!c0uEww{M}P=t!VPx`I3?`CZ?K#8jRnhm%*( z;ojeDfxn3=c5m)Pn^DFAWP>g!bfNNZkTrg~#F_Kpjs2}$C8osG8z&y#ayvzmPfCeG zjOt{xBwg*gYcuEAqL()Wf#T6I5%ZS>5shYI$3Vg=k;*^IXxIbrkN8J;Lzfh&UB%Be zaHgO{$L(tolYNO^A9zCcH!c=9e^D9eYBS8hSJC9IipybL$e#v}+=hu*N(uUjAl})B z!-7xYST%p=b*324+zyjRDFtzK>4? z-pNAP%!*6M09)G#VWLoy3$nf*Vyw6hRSp^#4aQ6*#;p7*M}-jVev|;yVMbPCUWChB z`V@m2y9myTKK{lEicSPg=fWprhS*@mCSay7yCn`nM%B<7re8Kj+P?U}-7dd4r)=b% z*E%fcATjeb2@MBDyecX|aG?y!DuszN={r?uXz7&J8ys$8IySZ~NN<9rcFb9tqZaG9 z;B-lzs!#52?km~~KX`E1a7c(D+}l^69Z&FF3F$Z%Uc{a5ol@5w4fszfE~VZ0H-OQ3 zrwFbJxj~~v=aCQ69tv0J-Rl=}#qdVHcaIr=*J5!1RKM|9jGry2KgG=Dlvsb(i=R1} zAcx1H3dt1pT4Djq0Rs>cRS^V8bedINU|L`R)!oHnThC!$*^RzCErH-MK$O5cukeh# zSJ9j5twm0p|DsyK;cg|&m}77k$k9aN-aG#Jq5R$3 ztvfwuU zQmuJ&aOT`oP&Y}j6wO_+AX)ML4)b)MafMO}a(%&u#^e4!=Kv$^q415}Mg25l zt#0#KNBy6!78C4g{9ckD;e8$prtqBie`4(g`M^!eHq!8crz9ZpCi|4AZu~a*K>2$uRpFRWbf# zI`8o^F63l?_z{||lZ3i2DT>GV;qa~!6dXGPKQ-TP3lS!SJtm*CuvFC~U*cT$3G^}F z9|Cfb9orDbvI{%*NK^xvL|HJ*62pK{f9Q5g2qtUpi5Wh*Y)7#YF8*cE$Ja2Nw^8o}wv5dVao>S}&PrA6pqJToR`+tZgopxbF(!VZ{<8x$qwB~RwTwP-~9bBG*!$xJW~=j z@->3O7!l0gWKsKJ*DX`Nb$2jF%w~%Bv7F*XbbHQDrlnyDx3^D&X7`{ySn7E!1qBUv zS;RVSJ=!_8a*!J$z;{?wv-t_F>p|2-vDwv?rD&yGePp7$fcw;0&f4t|!#dl|Q4lbDw!)nJsfpOKXy zNBf#LGi6wwwCbAX1Aai;Q)<_SEj*o;c)81)g$`yB>VEW5ypnHUN+sue%=p6;KoP<1Lu5i;%Xq#cQo*tz3;Vs*=qyVYwC(Zpiw zN_5$%mB0xBpKN8d**azHp2ERkxu#utycTr}(F~g0^gJOQfE3B_^k5&VMIKrAQ=L52mO50^r#VNjN@sw)Y(u%R6f-=9b=HCI(BBkq4IyOySO=bp z_EgENP9DOgfRXjCFH{hWbIr02b{NCfsroHrO}g`ZP@+a>fu0vz6U$ib<-d`9tNMfr zg%(a8l0xg?C0}DHQ!Zyk58bk{sZ{L#7fR5HlRkE$I1e3HZA5Jcgxz`H*J?J?&;G47 z{%)eRxEz0;+m{7DLR-j_6l!Om%Q{Vp%mz&1s`BzRAEy2(-bfBZ;6Ty5Ms(xOx~*gt8aVc(`?z=G9(B0~N<#tfSHNaLh@i}h=6B=4z#5I?U!NZQhBQbr ze1&3ueu`BW;wM@^5btN^S-pOt-^c$pJLgt7z_s4@M)zbmQ<$F$6l4@xa)GUizx`d# z#AWZ<@}GQNd-kYu3wBy4$(VVb|3{)a8%GmCz8{>MlJ*fOck|1pM@@eY=8wV^V3Zi+HMGYX%1rN+ z&bogm*SAyob~DF;GOCI|>~B+!B1I#2`~7}jd3u0uW8ip`+KR#13XHIfZ(a>* zE-lBdj0K#6KW{HoiE;4^EX_C3<&z_ zgNtqN-MX(zha05WQcgK?vhC zQIUEt;TNadg+V{SVA05}on9G;?t3k4ON=^Gm&$4++D_;tS;kr_A`+ZFJOwC99YP@P zTcj|Xgah@ABRYhy#qPzB%<8>w0ctKHPc>@mY9nUvYkOkW>4=%%?8ITiU%Iw0|2x~2 zv+W|^sc$~kHmuPm=w+OR2ov9MknIFhoKWwXTf4DS9)!TN2nrv|c5*s*?D{|iac%}!DRkPm{nNjO z`hr3T+7^AY7K3lETsUaVndi%Mu#Op`h!&7mNhId7i;=?c`fN#H+=N?fZ%Od)soi;a zf@a(;6XBQb)}%|uk;9oJBv(l^wlSen)1*3;Fk5_* ztS=AX9c${5-PNb!<+ZSpksj2qenT!>zjZqPS@Z0)nau_am1&80Gl@55Pvw9b z_OT#yeu}mjCoZVgw-I`ojb2sWT4ok2lWHbMm>kVER#w=cSeYw@!*f_*Z9p(PIW>{C z{15p>CZBHdq=U|D+?GBy3cZ#_k6zY-0BQ+9Yl!5XCXVbUh@;w;l_xtYYuRZaEEB7` zKGz^n5sJpzO;6i>LNJVGmxA=Iry@eq#Vo1=ys2Ayi`*mE$H+4*xiD1M7!2;yO@gw_ zWIof|@y$@}Fo8{8)jB@K02Ny**gU*hs<2O&S0j-sG;^O{QI@_20~+)z)( z_Vjgg3#+0x^&dMJUbdp*WOiMVI5v8j(aHe>jzsGSTlEvssgX<;wloYoDfAZDrY_zq zH0-YmJ4KBJd2JjvDdipt9{$+nR#vV5*Vz1j2hu4PB6(=;A1wZP*Qih}(|?;xE+0!Q zUWjLWXO|RQN)0RA6RGdlCbEGkA-$i|?bd4_jBHlIbEvXJg+0vhG8@5mdY`JQ%=DRL z&^))d{Z6F=l#JH$%HzNYXpgPkpvtj~vOrnEl6=G`1dn16c7Z=Sr>J2tcf`{ze>J-A zAOn=T^6|aNA>qO_d(1hgeXf@84_`6sbd&KZ2}FIPruP;32%C_0mVSsb4VE(dMsp?i z#>$iB_0n%heebWkCuItLU$;xI!hvbEeW#+RBXN($IfBYc_9rWiJ}frAX}FBKO)qYW zJwvpx3v&h=yDt$o5;7g~%1mmpnwB4s!F_f|5U~eB1u0}9be{J>UMn~o4cow%M zuf5NOChXxN6aQgcvo(O2?vY|fU;^9oPZT>XiD3ZeS?K&-qjvaCA|2YIY-mNx$2cB% z1WhuBwW|^qT;iZIStj zQG3zXy6KIoeuWk6dU`NS+TQ8PuxO*v#p-|qU?7K+@sxJu+RAlyzLC>7ODp(?puHY% zSI4H;8!@)OIdQ!?8&7(b?dGl)rm}!2GN2sOL59>J@Vw0ichnr8k^0-6MFG0ur+!VL z+L#q-?pnUpRnf#H9dcsI2U_dsrmXu*&w#T`5}gt0%39=Xl9~qKu}A@k$@x~Q$~+F@ zP2<42ZOeb9{<|Ln75u})b>yQ(z#0288b0Si@+pueL}k>jJ(;81`m`-$&MKXJE|w^ildSb7k~In#J8`2 zOC`2#m!wh0!txNNDjEAUmofYLacyqCB-PB#`(jQ}j*@Az9`316=q+;`yWj1)g!RVT zNT!D#Nijd(tKQ^toE$ZS*!Hn0>oMOYYr9+W*{@C-~o17>5@z3xHTDQB) z)Z{$hRaTx+TV?ZroIEW--)+fJr`OP;2xnqX+slyCTW9Nm_7P8b9BMQW=!zU!XTX&! z=IrjYFT!#tGIZhm9S+ciWAKxoW|K|n>!F10YD6wW4Aqv^264MPM&P*RY z>M@pVVOoQ)t)n7!(^W#8S2rToY(qi`z2;2Ja`F0NFNJA%z686T0M*?YH@`}6bfRZB zNn|mZ35U0uY%DHb87}d4-cRZv&C`V1J)Y{Fm4q=kpwWzw*o2bFCvvx0a z7aK{&OfoFd+oHxu=oY_Q%DDaRt+&!u#EexZML`~#b?gkd;9pT zaoJk#9HiZ|9TZEMWdC5FL_ML^f3YWuDV(TjMH9)o_-XHxbScN2I&o%u>*b^>xg-}k z#0D`wBVd$Dz_Wo<&lP)*GlURYl^N1^=t*gJ;hP2{n|)-R0X8)VVfJ@H_9-sreNc;pYgb5 zpq3m{-wvbVP1FOW?|#R$J0eo5n>p>p=F+4{^?Q!pcha41KRC8KT$Z7RZA;SiUwR5Q zA_(*ObS{Tlvk=cETCKao41A$K}I1fFbO zr}5xxW0n#g0LA~G)n>G8)n0ieF^9kpJSlNyqfy0p*YAb=_V3*hb%Bn?ilQH_%pZ2j zVu_MJ0CxIygykOvG)-N-BlO_yQUX+%Ly_8Hjh%u=3032F2wKJwl9Shs??iw!<3>_&eMV0Djdx3ktjWSO zI?FyXI`oz%;@8ZXak0KP)ItXs`RD%xeU$PH=YzSdxMQ@TUV{*n3gg;MWL zrN4KImt0D+#d1gM`P3_h1fQO2l-y$wP<<2#>%!SGM>rU8<0ALDH<2th^VK2vFoK%z zuna4!rj7Q79aR#YY)&)Lt%h%RDEm-vo`BM)&De5U&jhhBy~=mS@I>QtSG$onephemwk7iJu0g8JT`o9kNcWY&SCQwOS_7smtZ!Zt%)UWAC`Jc+ z!ZoQI=d6&tP_GX8yh#a3$hz?*ltE+{GU;O^qbZsF-I_m_YsN;yR}-mD+4XSO{URol zN)<&kkv=nNuwBg8f-6D=S0c(zWPw>SAWzNWYe(yYk`-oF^Hva&LEj2a@K4MtS8@idT3&WuHip|)>N z6qi;#RBJV-#P954t-~`#7iAmM5Cj#ge~kL`QmD8JJsj^S`JxF_S}r4+4qP0j{{)C2 zsGTIc2Az9^5mS4!>>kebEyLctJ(rv1^8rEMb;9f8`ZdHvM_!#-8M8++t$Sy3EM;@M zmY2G|2X3a)AQ=I&BLfXXgin)#Z+euzyF5MSElUow)g@p$T~3GLQ*5VCUaJl)HcZaQ zW!EwVS5S*&Xbp&Iqh>PO)LgSC6L{0^O+xY1A0uMCS@Zu za)%bX2EuPK+!FndnVGU&a9E{~#6)sPw};-#4Vsrd;N&b;AsPiFb5ph;YuM`UmV#l$ zPWQhU4c-*o3#P042t+VH=8{dhD{BQ5RVS(D|M>(0*Z)qeJS zY6)YUV7!J>&94s~Tlh7JpupsDj^29zPs+jhrvv3Bi__^UoRU8xoHJ zL6`3gs!j_l<8zT~UQ}PQo6LnChDM}s`_~IDrT7p?mL&^@0e93Cm29SKv(zC;B;J~mi5LC-1+2CzK^ZGF3 zYuluxpTF0>ASMCo8dRzg8mDY+gi&?p1QgIe`C_L(R)grMdWYvAeIMVr@86BmM8pe>j)=Hh|Ur=c^B}8&(I^E zP!6!_f2l&fUQda+j)sNE0+z0CjlwAjaXk=iy%Sizg^$3v1QAFZxscX5-Y#wff1%i2 z-cO0Gf?!sVf!}*OTO)>II=zQALCOV?nU?-pLAGCkn|)MdZr`=1O^NMs1o=cNFIq5qnIfFZt;sheth zH#REo_|L znO1dJDN);_^?WSFn3!p1Cn69S--Pq0>}ok+^`L2!!VC>3ptX5Jq9`>tD~npuHwEAi zu~eYdb3~cgGbcJTP5XGs2m^FggSm{+Me*KwLeopkl&2sm{A@zH;cp|tgpNu;oXqvW zXu}+&U`KY!TpFzrd;yFGpjfRIU8_a1q{goB(=ALO(QjTq^6xnn=jH?gwH`3u%}+*%WRc76yg>GoHK4SlvwuJ>`VGfaeM61f7+<#u&#@gwh6q9@WK>qjajv} zMjT!F=Q;=D4vHUL##{H-z7^`IdC_m##&H@x%^A2|*GAY9)hf1i=bCU%$mwlwDc$}P z5QRr2qI_MQEQwEN?nT=zxmsKKoN#)M1jgCZk)N~<1^sUx5p(xx*k&y7HnDP*%U>pq{72GoScX- zH$G}h)AkAD&Su~FvmG+ovpkjqvXYpoiksTka4Y7#XXe#PfX)kC?uEL z*Qi9#;Vqp%k`q`2EI?~K?}IPL4?|0-pN%a^LwR3eizw_3x)XQIwCZgjMaB|>+K86S zv%t>G^_v8<8uO9+w?%L{WWvx`bL;N%IW=xQkVmali%1k&T19Z5ImiN?UG1}n_Za%z0Bxl-d-fqt7KRdM;nqk>of8-sJf&IW<{P;3 z4y8d9N=j$RHf;-re=ZhW=gX&#&J%64rTxz>jV1)3b=hHDCG8ouHQo>yLj7>Pwzi2X zhbGiV2~<%eM0A^DEtV~YI`KPiyU}nSm<;kvkj}M$GiX_29q$>BcLY(X!Bm6h$&qdj z8GLZ8o$^}fL;VWj$W9Cv1(|p94+Ot+U~;xbjBq~{F+pj^7m*7_hwh9F-0Ic@9-|AK zhb205jtJ6QYLISL*R?_|<*Yvh=WFEUx54v|zQOB&hCJ&fiuJtHbh>FR;i>DF%-uy% zEUA}CG%(QavO;37h^AfEA)1<@2WwidNAhA_eS7@yDO6+!P&bwya(TeYP>U{%kpdg< zApYLuNyxAg;}j zPud<7U8tpkY!*VMqm2!w_bJ-pG{jmr@xKqSy`H-O?ZspgfN`shw$nYHrW1NA#j}oV zuS%{Lyw{)DCKT|K=OmYciAjLP86bmuCeLZ%hN#JEE5h_lPSv35l%elxV4yUbVkew> z8D}R_(Dy1sF-*2oj{T!t_}&#O>5eeUL+hnmzcCZQKMf$QbpqA%r>a)ci{VVeY$hP? zRL7}A5?XCPMn&mMDk$DQG1!J~Y9D};$HmgoxK5ttaUih65v^AqR@%C}qz$-B*4M9M z3ENZi?&XLh9#dPd(YGZJ4T=E$*Y&eSD0(z%7PoOl(-)?9b@M)>#lp;+Avc{wF1QAW z^WYU?q1im*H#C;u{q1V9DW0Y)8S#4GrN}fIsbz;%Ii**W5&UjkJlBvU@+e$Gu-Lv)!n$yeQas{gu+8AOt3oxBT)W z1hCIMhREPX|5$wBM*=Z*u`@ zd|7u|ACUvp$l3h?L{R?V(F%l;t@%wA+bOsnSmg=q^ziE+illmjz@Lp^{27`y3Jg(4 zNBu^`@Rn<^0+{k^&y(lUf-B$27;sP6)%RJBF5TpTi?S$3`t|+{F*r7GJY`mivjT))c<}6`d=m2kTO8iMiaXci@#=l=a+@V?Vf{SKzZsi=hCkdmX9Ls?nz4}8EF7n-weO9 zQD4=h{dXkbJ}-xN2IwzomPH(^GY=FZ3SX(9msIYxTDNLNprGP2_n&SFBq^!tQBqOc z=39gJKu4WcRA9Ot-&!&CtWhp^!vd`^_a#a~5KOSU$uWT%#y=?k>LIJolG`XESQl9v zzV+2SvHinlFZE;x_j|jxwn70t=ILg8GNXmynkBk7WYAVW+@k0B$)mkw?KbOLx-Vzd z>-oi+OS|U6EtZ{vA2F?ETfK&RK*|gcm5=TZTVpws#TE9#AV11oXR@g5EO-_^iCq03bp?5X(5_0+40Hjm9Vj%{Mz5cG?eWH6% z=_0*1IwS>w_JNa4K>;r%-!&P-Gx1JrRve)@cK-pb$|H?qZQ#yQH0}9NEZz*)pZFkR z629N6t(PBOwkRpH$1L*-r(t;z9OW1<(6XaFS^__r(>|N%_+DZHc!_}-iYY>3{WtFM znr$QBioOS)hSjgmdAr+e!Y{qB$W{h)5XbRSstlL zMv?KEii&)SS#Vyx{Ll(7?5RRCbVvoP#j}SE5hrb`MXxD(&U90dZNLTQeFr7gcN=tr zS(uY~w3g?Za;A$+RG&mIF;x8xu=3!k?KB*r+ZUz`!|x|LO^9AdM-1A3USBQYZS)1* z;JE6zdO-Vb}a`7!y~^qU?oSV$Eyu$W~pn+JKlQ#D68Bg!2F& zdoXQ9N9igo3w`C9k>ar6Z%R5iqR}6z2$zYG0|V=Mk*hN<_+|1o(wP9K~tbS2&3S%@Df}Y)5L;CF1wQCzo&+o&ZB3&E|g;gxW8Yo6Fs^?>YR=eL0(EC$30D z^MSyB*zFueD(=Z=9G0k8G$N==*y z9Fqdu%~l(P^-1VjG#ldyy)I9=b2nP00hzQvN3wDdwONKUqyKz%QIzwH_4v6ke^ z3FV&s*YjM*21eD&sorBS;rG8+HF2-+!P$|(Dp7PeneVTu=C)VdlXL2S^IE1XeSk}1 zgb3TK_SePF#~1YG0SOz1ACF;X(L;xpmTIg4%8FV4aU4yVxz!PY1%qYzM1B7uVnwzE zrqC~YQrkx}*f?GOlZ4E|RNCUp81^Ee{Q0<>L#g_3J>z=sm+oan2x!V2rXGv`gtsF1 z;6n*xTQWWDOH6gfov+Ku>)EPwA99xmjT;I`9IqVd%*5#Ze|IIor%#IPzP<>L)*Sx;FPsvyt((fkBYcaEtL?$*Ulh zcQY#%5{;nn(GpnPZ+|Wc8kGhIPh5zgC`1pof&agkrt3^}m5qC3(N&$=JMOSACSyU9 z-<{9@fRrU1kXBYb{VR)!WpZ7RsvcxQCW_p{qZd9_z7sV6j~%;OYQtdwZBKD;f#U8I zcPY>Sf3&!}y9al7cXuf65WKiII1TPl9D;^Y2%L*EduBhvzW9E#o?6vfDMD>fX(uPYiTYwpl}h{-?w`BOIS5e88OiT7)G0MAyB^95IEi5 zjhIvHaVZq-Hutsn;c$ykJv3W1#VKa6cT$wB?tQ)L$Hy4XPy?h_Rt%91DW@;GAAgzH zpA2Z8K$|%^cTHwjpTDuk$NkJS!p^Zf-Lvb{-P&;jAj_L{zAd%P9G$3xHHhv8s3Szo z>4L+Cr})0|@%DM1W$Mp9I3DBapR+Pl^J`uh@j2I}+Bn!)cU#*S|6gPc=WZV!1>NG1+hvL|0l&!*!=u?ydDn+0FWxAr5H zi1Y31rtPaf!lc?s{o_K)kGq_LMQQ^QM4JjkN~cEa`5iK?_Mr%kUXeTWyh_&MjMn4S ze#6%>*kNp(*Ac7+=FXgk%}vfzA&zg5-#u#zBy*E}%}|7H;@9y2wy#z6NQU95>rSXl zmCGR{c_LgU>CjyGw;w{M@wWM5jXz=(qFR%hStzu}0@__m$T4N##x7cAVhUb9hCMC> z{LBdy5BpElu`dTRf>Idty*JNh$J@TH`@8BUqBdj)gpOJB-Z`;riLYcm|Y~%Y$-kL$vWHp ze7hgVDeK4+cQz+`%EZp+*|A9WxI@1R2uPe)7oIr9CI7a2XPKn8Q(L?@NmgYakbOBx z1~UzcRC^V7B@7L>L7zs4pK_wMstMJqHu2^8}YnfkiioY9*5aS zF}|IXA)hD}zu-RZIvDId^ovI3HH*V<`yKm_zizOPin?;uLgYcjgx*%Y?V&dpqD}L1SEK4G_vqg66 zzTx3?sVXM8QrSgo8NcJZub6t>KYPTzjPxGz*N*YFD>Md2LmWFqG`4&`WDNqz>$&uP z<0w3;jSer1Sp5bxuZdT7L>%O1!Z04_gFiKM4s`#+JjSUoL}gF+x30YV;+Aq-?Y;U; zW-N$z0V1IEDWZYyY;}rx8B+CZ6J<74yO=^0*nQ@P(k_c6;b+E!Rcv@p(DGER*9OgBm1=W09Jh_1 z6ns`pPe&TAE5kd2V{fL=DYt;)${fV8zhBUg_9(G|#@{5x(E?P)?rc65o7t;aBto zYWP16jGhJ_ij6|^rVGs(J3bTeI{q_p61{TWi(5TJ+03U5y9MuLAq$VfR5C|IvNkv? z#ph1vzcjw3GQopV9P*eM+VWzy3r$T8FqW;$zw}MZk8#p(ie8-x`3DDyJe&Rc4wz=G zl$U$SQbA6F8w$mFb_w1cehugX!oK zhB)IU+h}M<_dkBzy2B8g>^H+9wb>_2{%_chu&~n718=opy6{K(d#Fa4o6?wg(nJ7q zEh}a_t41WZUT4e5H83F3&R`ked{2S1WQ?Hk?nBFI4L{metfe*2W*qLU=qDqzZ^rj2 zD65^Y_Qp??ryHY9_>01fIOII+qrr?V2^e&I%2;U5(+>oeqf1Biy3Gr)zvn)KFO(lb zTMDfMw;82wYTx1OwjOsOTsyc?4+K~_^(_{JFmxgYhK9MR_a68^V@+nrn^$T}qqHk5 zwq#DE2YUo6Wk?OFSC3?5$@yu7TIt;5UN}rhvH!$|CgB3p8yu*l3bO?ckkb5g`+cdy$wp1ac0Dy)^eWlO-|+ zqomVwk%(a9sSK^sUL2d#0FiNCnNzBJWk zy_FOB;i(IE3TzNJcd?i`H7m~ofh+2dF|%YrfS6z1y=6~9NcMg3R`j|RPnS6Y!pIea zxl-(Ytj^+$K@3E%tohPwHzisRHc96vW<7XIw!2c}=3pF^|G?AnUwyg7kuryIZ|wyE zQNFDeLSYwOEUj0Q7R)Oiw^DUb#5$_xU*gQNSxQTle!7P@ss$M;OC1FgS@oNDI|q^` zlAa!aES^0Yd~$UqzOd;eH-X_r#Wz@ z=FCELKnoS|HTMw$NWPYvmvl2!`W>yHiqk423YGC)CCR-+CI!?z zwg7Hnf>Wmd2$hF~MN#o>8HjB0$I)y^A)m->S!HN)<0nN3(>xHFPSVswK1PCj(1dsR zo#raaJg6{A?n%&#Ud%kNTg^0qUsGa!`>{97S(bWH;Wgj0#1_d7Uv43Fc+k0=Pz3~$ zd7pfKnA3WlQ$R1y;1;W%8Ahg<4@whbZn|4s5KD0ygsPL~!1^q2S8+t4R za;BTEeca~bbfGY8p1Tm-XQ(HYTuy7$aPIg;S!Z85A*&(SS6;a>$zgclmeaK5`;Q{u zu&-{Nbh=V_apkAxFNy>J#QJAEb>vL}*fW4f1)|sgsp%$2R-r5?nqb`cvspa>sALzw@tOj;yi&8*&!lFVN_ zm3?k-Zv#caIB2X@C+TbZJm(GgX(~BHq5G?BCM=ZL3s=G7=eW{n&xQs}y_v58w8=*yJoCk)9&TTIGH8L>U*HUOmQ(l5j@1{VZMLH*eMW~ z5vj0aSSK=&`^&*WXds=)j-0KBF>e0IxYy)?ZaLW9pXyX4pmZc$ok-PZK|ugO*+`l%-eYf9v3{m!UVSeAdO^4vTFj z;Rdc+{5RAYO-P8p;(Y5*(&_DKr{)84u(wg5+BZ{>Am>0j7GZY1ipuHst4RyXVt2Sw zEL9m0X&oLa{D8-%FPc8k?`j}4C^S@6`yjqwCW{ecmQz%LsV)qwIUxnSs!K>rOZ}v9LKavJh z1h!CCO}C7g4G6-8eAWoUnSI#6pO2} z-K7Q_nNi1g&O?%iMN)zxBqXzVguJByjmw(NjV*Y8AG|@Eu#xAse}c)_FYna@IDy9+ z`I`jWBf9D@gB;zh`IxzhH=$Jt<8-p7Wn=z}d2=bD;^=JpFC8c4j6dwlJGogstq0Zw z(@OmHEAdm-{x@>Dt+0dJ_9;7Xl`?LsUT@i@K!=dz2YZlyc)alsRscf{>K=z(^g7^R zBj|^H9ua)Mnpa;<1^Z-empI88kf4;pIH9kbu46xi?{PdPDvw zA1j0-?kJZLb%izVpqr5i)WncR)iNf10(M?b{8ggZDUV}lAy4#2$7N$4j870NbiLxF zZ@q*=X&xnHn{V#`mHpjuDtIw-D@&^1fzYXN-*tr`Z_1Cf*6L@JNFD8lN*7q zb+hOq-mFtwVHw2xhK70e*5~|2>kg5751x=YLal!=z znR#Ha%?zoAO?62fX^FnR1}}Zb09{6|N^39NkkOJkzGg1Q4E}!*+e5#kD;HYfWg4$>YR7a_rDNuPIYw%9K6W|b!mE=Gkv>n zE;j+}4ZivP)AfFQlBeS`eLD+b@&;8B1IujN^7qq&aq&_Wj_P@r%i)!MhQaKdnaBrvFvi7+M6N4tu1&zP*5nVdw^Md-37rm( zVz-<|5U$n>23pmJTH8+Onk;f_fqm;e>ezavj_!UQ^R1!upa9Y{l_V6ljL`frZj=ga{=x z_k4rM!U@AHXpPBd>yx0t$yUuR0W3E_0NLlW2pCRH*OmK+X(Q2Fg;Rc<*TmhBDeM5+ zec`l!s22?Guo%a$q+`u*o7OJ06tWuE2D7g*U5p%ct7TOm!A$H;lnPMQfABU7Mv*Lzoco&wqvStX~& zi=G6ZQdMNF7aR*#Ijj^ev^6ly>L{Uz0~Ga~4M%o}yEx7RIMu!&{F437F8Cuo|NN8^ zBt8bz?NQB7k=Bq-Q`RL9wN2|39TvOUqZup6wn`JVo8KMe{pV_<+Sg-=;H0FeF*b>l zyJ=4ft9y(yeUEOiGOm5L&R11Xyd(V?$HZ7|>Zrk8$NzjusPCbEv4)Ql_UmqQmw{r` z*V@`bJwi<~?ET5$yXPMrM9lVe*UF>r%(704=B9^6It%a?Tzct}o|=OAj~oVE4LmM8 zDe1LlyIJ}CHv86%LfzfI`=OyU>Z8*FlF!f2^z~c1ZB*>v7s#;0uK2A^)I;M}j^)r| zk&my2vk<>lD0=dTCVX?k?4si-eRoF;-=!Hx8b;NcqW z0*FRL_(EgX4S64u7ls~G@bpn>jTL{ztR z|7-QoWW>=JFpr{pa{y2F`{tC!Wmk^kEQ(|j0(~aI^V3;s0uq;M+s`vKg{Tb-xM(Rx z>(i9kEC58Zoq>fmh=ty~EW!ErZ)ry9%;+M%{JrG9k6Gkrw?#4RqvsryQ z?1h^P=Zc+vFjvUpx`aDVJ-XINRj78aBI3MABSy+1>ZN47b1Z@w9YVzp;OvT zVuRk2Y_p@N-K3V=w*;{3@7!&cPxbS8b8~z;O>-QNXEUcm%kA0Wj{4)HctzbpH$*d5 z>Us?%Jq*qIJ$DQl>dha;-eOZT;QV3tet6A5CAr?Q%cT%j# zDM`}%v4n+ZE@c2iuAzf(i&gd%w&L}*2Ol0YkLO#Jf;ZS@mKxc{xl zH09K2AF-od7Q~+@;mw!iD+|kO+#FcWxHP;lO@tL3_rljy*0p?`GAjh)=99<{sziu0 z9>xjONDQ&@gV!xm`AzTBUQek}5-4=1C7L##b$x18AX8ehSl)22OdXpNXEvKI*d#)U zpES;T6y6*0pkHk2vy0p?-#1@FdQ5Ne5h7@S{jTP0vE6HtbBWI~T1ADQ2~Q;jT1TH1 zFqT+o^+=JS1EAu^C=g$gaX6Af%`_epk*nY)q7LweF!3n@1sB5Er;(MU`^698en&K!KH zdsi$~ZAos$K20!{kQyI@DNj%0F5{K|;ZR}#t+(IS9TUYB(eX$c~SG{Be??yWOx0w7I*$-09QwQDf z6`Fk&^ZNfj*T=J}SZCB(AgP+bVM2p>8Hy$!hCV%^ZzO@kb+9AVXWS8sJPSGWXoE{x zwN(Wn-PCQGQ4YJ8wQ=hjt?TM;AEGOEqKdH}E{3r^a}2-nmZmecD&i=OCW%nG-bc6$Zg!1+b&3Jp2^JduG$fZh2L4&y~u@r!AftxFM+)N~*k8h5XQmd|RXK8uic>284Oe*6JR~=oWv7;9zlT4tpf^}`UeOS{;t-ZOJ@)2$@+N8_Q9Xv z7_Bm5u6EvvR6rDF(cz5@(7g^tJCL*6B0Z#FvD=k^+5A^#+=xji5DL_-*v95^Z@;z) z6f6oHp32wFYG@E*rkJ2njuqqar>j>JsB;P&;aK)g7=kUQtA)kMT6Q85lVFbe3>=^s zF3NFUV198_Fj)iibNl>}#m#y0&OSrEr?%02Y{=Wi!IM2U`Z~IW4SegGdj8}7=Prev z=Ud&v-GRc=R$#9vuM_rRY~0Oc%l=Paa|JDIFT+Opq#x-Nc z;s3@h?~$WyH>vnt)7Zz?%8(+&(N}%EB^iw9fb$0?z+Ac8Sd)4lWZV*OCDZuet+O7W zTT!Z;K6v7<&2^k-geC>=!9iOtxR;?NK5=wFC&x3@4ODm3ue~o6Z#yu$eIadZBV1)) z_8OnFpSI2WoMN*nUv_-)N*od3CfHz&%~#M9PvPheAa9=fI~l!k zh4F9}DtH@d^MQRox!E;i=wnlt*)K(d`hnWBOZEBe!oHMvoU%NZayF7$nYlW|bJPQ` zb)RViMYc5spqHXf-hY#KBCWxp;r7RIFH>LM*OI3%g_d-e$M|T+mD(Lxi^DJQV4;!O{}t9LX6g!gyP^FPrD=&T6;^FROd VKmQN={{jF2|Nq$oa|ZzU1OT^CI0FCx literal 0 HcmV?d00001 diff --git a/assets/rancher-monitoring/rancher-monitoring-104.1.2-rc.1+up57.0.3.tgz b/assets/rancher-monitoring/rancher-monitoring-104.1.2-rc.1+up57.0.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..36158e01202d514058a4b3f991b126f683de79f5 GIT binary patch literal 478538 zcmV(+K;6F|iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYMcN;ghAbP%zeg%f?du*pwMN+a7o83A0D3+aQ+7BJcPM_Se_^$tP^C5YjYOc5XvI^(s-2oxYxi(-nFN>y??8Z9U^&v-ei z1TTqNl3FGiFUy)ywHhr+Q6`xNA|tqODyam)B{~uON0KQEdmG@qA_6NeMi=#rBuzng zMy}=*4V0+KUE5 ztL3CB_(~?6C0tI?6=gZUlKoYwtO~qJ%tF2J1L>^{Yj8<~q?}FBy?^@}P13zD({VCJ ze@2sZf1>|+ueYg&FDsmpDKa0!Nz0W~r0i{15m8blVIBo#_1oP|v#A8Xq&bnv;!;de z(?gnZLAcbLn>Kqmx}5YvaoZBjbT55uC%TtT#_K04NI|e9c0s#4yr$7lHIYgtSyfNb z!-Mg-e>Hr)l9XJmrs&~#JbpoY>#9|y7rUsXA|g?o7roVLst-`b8DT11vi{2UAT>*K z@Kria_jhnbzjvQ5_jYhqH9yDcSDFOmq#`UQETcqDcM_CwmQw{)76Q*P!zp2SR*-xL z*>h{Z?;s>d#ibtEYKrExAk)z(9hs9paiJ?{o)K}`)15E5p^v6s47i?=Gldm-L6o4G?84WSK2`tNPlH;Sh!%hz?R%eLzaYW9IY5&Z3H z9egO$lta-i^O$-*30=8+U!}rf<3D z{ae0q$8WLWYxdjnnQLiHtNez3+O=~cGeN6{Y6r4~z9+K_lxd8}lth~A5R51yQHdFy zYYuQ1eQVerlw-M^aV&DVtA!GlYbnPVttCLFkp0m(raj2M+1o zY2zL9YJ&ryA!O7nK$~()z-qjwxPTkgKN2f?_7>8Rh@J&>!^t z8$Q^Th1Fq|78h8kCB2Y+tz22tsu$7zT~euNS}EW1+ZS54W#k{pnJ#Mg`tLtp2u`XC zlIFy>ao?KCGUH19yZCapiiQQB1zpTmP1TYLE(?5_E^+>kB_)?V)A>(D7b_7g!A~_| zin4S@rR1!q#ebLz-$41!T7gW0z5%xHT@(#BwOWh0MnJ<=RnQF1;*OW9CgX?cB;89y zmQMayS60aSwbefUPmliZ5*Ia*A5a0fQU7|EC{;9g6caKE}|Jhl_uvo zqD*nL;w4c*a&s`a8Wx7sdm&(fwKxw9CiXM+2k73bI|MC=_)N-l$wos?I zVV&&&M&YXFATYXr>d6AO7OROvHMu+Xu0ROkb1qXD&ho&KN;^G ze1(Q6x45l1k|e7If$1SE^MA z-7(FGP)Afu(TJ#Q)SQ%<obGnHHm)Zj&$43hZ+F&jnd^ijxbn3aVuINem>zL8&Cq zDXZVUs)7wbz~p>OFNt*OECeP$sZR(# z&k4d6{Z{b0f})_hAWj>E{x;AEA|3F}Fn=AULYg(V2x&SdEU!5AJDk`(ZmhJhp`3B? z9M4E=dju);N6Hq#iklU2wf}a0Zeg8QSS`=bn{`oGEQo@NzC)KQpadhzQLv^D1VEv58#61}&S zi9ZpsyUmSMVyWOp`V~2ewkN^s^rdyEcfNpH%9))6hDV*WPV!&G*Cd-=Q}5p8&bH*5`h1$o)BS z9G?&ecAQJ)=gxNAtcm>8tMWf$xWU8iGhoMy?3TIbvKPMo8`u4P5^8siQ>d zSzi&9VTN)lNT!Mv$^^lRm^E=KCZv{n;-Hd{umFB#z}-)OS2@L;06y4LemEP_4GKGLH$ZCoPU7YdcX3*O(T8U$}u2PIAef^!u z;9Ih(Av9;p128~KP3&*tX96;T)|{@0TA+|(cu1||ZkX$d4{Zc`es}io&VF|#?#_Pq z?1wDWkw3~e5`y5=ZA%HFg8?BJ1N|hYw61#=jNv z@Gu|`57YfS@^D8Ux+`%<9)9=aVF#>khu)#-SGB$E4BM$MK>5R2u|hM_Yzdu%W+X_R z@iokOh>quO8Fmp>ux1)xV7JdAholDDYz4(my=@^Q4z8ePV=>wp`jts$SdtvMIDDgh z$~lps%(4W)Vv13an}{J^5e62MOyG(*HDbEcn)17O(fLdv@8SS?=R$XD7rIYs&$)A@ z`{#6}8}?@ASxXc9vX6W~!We@8gk&+C`pvZQ@mG((j7eiV*UB0K`Q;`8`KubGhC}qG zS_qsIxl@~e=s^sJ=(J|Ye(`A%W@OIwIyWU;AkEgyaCU(%DMlFm4?aUn&M$g`JcmPc zT;c`MJ5o#dOVoycvOVk|M0E|b5Mf#g#|y|MDNNvo!VAc;wU}4k^7Os8Uu=zUe zjLZz}u;Bq6j|}E$xQphiIP6rH%qhza^JjZ*crJh#8iHc38~E&(y37$+8)#|`L~cNG z-8k0ts86Ck5RxdvgaYS;u4qvpC053z0*h=3@xc9n97h|CD3~~AIihS%OnW~<7Y{9X ziD)yLHGTM_OEeIhQIvz7aQYjPog>Clr4~9w;l7{u-zIL=w|Dy z8T9M+HU}#PvDj;SXzzK~THChY+34ZN*4A&Rb!_|9fWh+>M~oIXc@G!=L5w>DOW~?241(jbDu}05lr4zV#5IC$;G70` zEBf2t?dm27XEr+xA(dZpK^Elg6y4L@@cI25Y&?>v>jtZ@E1qBb-M`j%!!;y+o7aUT z>OuO)dmdWn2G{NM&24M^xA&f+STngk-T|LCV)A*#^K(6R7}DRH5t9nr>*G)DMfqR% z(=Q(U!GDdo4h_nh#a$a<4?r|Uf0#5lZCBlvYYrvAOqVpLC)2@+&D?|G=xO=|0Ki?W zH@}cZx*zlr`oX9~q=xeH)$6BIeKw*@15_HywhN!jS}IgmIaVeMK~EaEAbCJ73Qcvw ztOQdu=!&c68up0`3|r($rAMfbKFGE7}4AF}4 zyoFf$9Qdw=5Z*oxXIO1eRG)03eESNu3Pr#$cUuz~QnBKV{YZov(Msugf zAI?X2W}JdN4=Wmt0uDwR@bkv&%>Oh5+{~AVP`FzVMufgU{g;bJ@(H(DwVqV<8;KX> zjAWeU@-yE~kuN!WdC^vM`G#c!d(`M_Zle&BQ4vYJ`4sr$ceSGD`TwiE&5JuwODamUs&RDNK zE8OVOAer$>H^f!-(wha$Jv2pw^oxN7aYhvIi05;vDapAeEc!o9eF2h|XLW^VQ2vYy z$IUi(z#Nn!0eRX%mML-6bBGR`tnsfm8kts&-o0r0Y7kLVxMrBK&NR4oI>2yi zG}}Quyu}!0MKU_4Subd`0Nrb+15nA@arcmCSW8Eo?T-#=w%PuQVX7eZ_{b1|>aYH7Et58vM2-HU82jsJ*2v zavIbLiD~C^KuSAho{)A+EEx?7dWdLWY5x#o0%{99`Eo*`k=qJh%7pX|C&x$piZvz&1ekYtd~`&>ViC%^N2C%Y z!;0kS3a`Q>&{hf-Zg&uHb!H!}6lwGfjfspY9*UBwmL-S52F24b+cRul>M2)=hMUZ3 zLG*WIsp!=F2rHgLiTmzFQvW-#bF=bAMsU<2*_%C%kdGYbU$2DEJH)Q}f!3Dx5}MRvljjZyi9$D8OtE*i*8gQi+v!TpJG2 zUyY;@zb6DG@8INekQnOM^>wiVO=x=bieH1+1z*W17J<)0jgV_7k+H1u-i%?rIbdzy z=MIF^I-{qTM6CP@fGHMqkxP6@5Y|FiqbKBZG)^b!_>bNesL7BqSKW8=jS2a6=A*sz zAbs?}_x755BV-6!>Zycdkh@3{vT5aYbC=ClBd^bp-z>IFY+oYMu{*i4A7gytw>6<$ z`fABkVO#eZJ8cb9!Pq;&(aQ$HR&`O>oHyur{*tQ`L8P$-HtsNu7VPw!!zY26ORbf$jKbH>$R+22^ zW%Z$i~NdwaGGvC5}W852%uvdqpesqqgiB*-c0fU&--IAB2k-Bt? z3`-)>eUdIv!82Sqkd(+>cY#A+au=m(`oLMI;ahWandzsLd#%`<2*Qlzn=^3mD7hed zg#}S=bYQy$UsO$lG+SW$+UZ_(JqV+)ya2QTLmo*q!emZn8fl0LfJ-VtFR>ztTdq+w zi<7!2EK#r)&Y@=+1{Q#ZWrjY0ayTuzT3>}7OV49pk>Eas@ZZ+e3$)2zIEmk)=4}!v zH%!z0ANkCflHgvP5p=H&?cRePzl58zxp%mMkCoQFr=y(>#dh_c!gSi6VDS73QQ?y+ z#^P;{1Vp9l1%t0{2Dj@6c5e*^hnk-b$_6HCM>d=YG71}xK+*_xNIT(~jXSZ^>h=C=skjb% z+O5&ESJX(v&5(@?lXY>k$4twa*0V7HouU&iCDnUsJ*eP4h{-Ox-1FVWPfR*AHR%bk zVcj}ow#O*_5EQxjZe5HL+v}+D%nJF)X_sw8$h}QtWu^97!na!SQUi$D(%1ZguxPU! zE)n3cawYRFJ%`*S2Mqi*GF!;=^^kzis;~{|#080z`0X2po3Gxq72#pL2qxDCz3gQbHl6th%xD<> zG)fbI2Y>qEU6GmMya-?lUhBUTedQ%2*B8mCMf!}Q*{ceWb@MMS$+>3oc&)4#0z#B6 z>5Qs#5yu=DaqAphK8a{Wp51ef3+QW<394v@i+jF?@uRxA>HmYQ)5A4obKbNkKh@2! zZ7Xlmu8d@5TbYZ=dob5VwN|WlwS=dt!j@ARxWs?9 zBru#(K;2KUG?LJEORAQF*NY|kjuhoR={C?PMu$U0*n%>WI$!Y(FfBhKTWpR)h+P9u zEy?O$QJB0>_BAcRDx??!=c{XXm4zMj!{#eJJAmo+)Io4 z&(A+6_E8cvkOj^zQrjj~HAoO)IRNSHaSf2L z(=Yz^?dhvGCn!NjL}6OUzc=ryyb9D`xWJ;6CN_pz4+n#1O3=lE=oqY`{r{ma ze*PJyFZG4{%P-R}&=2}2bc=rZ<%bq*y`DWw-1Jb7o-aZr<F%Uq``f*xmltwA!*y>rruDiz4dt zwDAF%eu4hn?lf+8J(U#|xHxBAk@E`Y(|@~Z9H8W~D9~SaN=kk2-8cNqSwS;`2L50S z!U%(b#i7B2p6Pmu+NPg@@7iV}`iTkWcAv~5g&N82SJ57Y0UB6j9iTygMguem_V=9^ zHMD&GmR2^P`8EcqSK|fWhUgg=p?Lz*tV|w3kd+N%w93FJ;Y*O}<&t9SPDP5`Rd`Zh zDPJ}bac6;dLadaW6?`$;8}E%rY465;qwGD*VGQ9J-Vq0blvII*22dkauNq-(+9CxXkhe5Ih})5QTYezAS8_2&~Kyc zwZpK#Oc3m&@uSJMpqqTXgoQSbUHgPEM6k~5!aXKz1w;oP@?991-iSw;RG67aLJcvL zKG1!#=npSU!fMI*K|C?DwA(cFi~1dxN_vx_by0()3^tv#o}9(t$UOzzUwrVVjcpB$ z4>>O}W$<*o>+A%prF1s9ITwDpjlC!Vqi}`Q(j5U%@^qHxtRh(oiX!8D7!FIfj$=|s zzWdu9ISmQkKD?h3MMOy%k!Z=U{K7z2LCmqyNaYV)cOUoc&4^M&Bzh1D^sG;zrrqX~ zT2;iIJ~xvoI@WTR?gozAnZ-rFK(C*lwe%t#%g=Fmu7}od#^Xdfj{rt}s?pX=bAz!bBFq)R>yRkUwU(KTgH>LiMRryI2^|7<_IJar?#%!SxW-EkA&&9sd#tQy~|a%=B9f_H6JwRqI9=| z%Pj}n4?9~08%jN?b`XC4xw}QF24dZ{fB7Z(`DgS83*cp4&Onk%*S0$XYVKH`#v3$n z{|2JjGpYtNL8tLU5_kK91BI)Ku-s@?H7vq%f76(5LT6YzpayuD^C= zZo0c-KF}3oFATtCt8{m9(s26_VYOVO+zQhA@kxAeHM`&G%DJ#8jtU6wM_?pr4zbCOGk(19nV-P< z(Bud+nPtdZC9sdWf`-vf26=Tta3?lV7 zymD))*|M7EWMsj}(U4dTY(gtfFCD-@yw?k8f`Tft5l8%_gIEvUHX^;oh|6)Usgiy3 z$gqS}OUi5QRZtK^3DfFDaQ4>0*)!90^FKamhFEZ%e}fAx=sCP?e?9}xf>X>E#K%=x zCo{(NB|_&$%5*556G??Jd4Uk*J4Q<2xt1*zH_79S^R8wX&B;lww2bQ$FotXFV7xXU zoaet)z4|fP>@Zi@$g?HBaK%@wDfp*n1S;*y7vUy9-rLPv$^+@GZGyOX>tnt%*II9SUYH*y9?NXZ395o|HjjX zrFEh6H_X3{(ZBvH`t#S1zI-?y-`7|?K{;MM(7WY4%p9<)wXWKlJ@R=Qy8P!eM=qG|>mOsJvy*gXaMi`i;}2 z4~YCNS8|VnFJgb)H3{*K=iq&_+Xdu=6vvV}IVA+{xqz5T6m!>25F_+yvhNN0u5g>! ztoMq3-W6xhKpFF#meeU}r9Qxk9ZCTOoFQg-PCmDT%zZMtNXo--DYS+xn?9zfA&Mrk zO58PWdIXuE(`)k}+fBWy27K7-$;QdqWF)=NjnHFflbVx!i3;PI?-ybE(I*=+eB!wx4c_dcS*_M){oE$x>wWgrCKL^({2NK8s%7OHSK6+s zTk2nXO@I5@Te~dD(>~_0;YxgQG+`qltP%U8eELoNp5_wu*_mrWS2K()!(@+F<%i$e zuv6(e-9eqfG6-tQy@x zBpp^fm+?i)a|bu#NKK}AK5xfyH5_o0D84^?LP>?-g5;s09k+o6dE~!J-SateiKCm< zN(_x}0b~_$e2gHn;>-l`1vzA}Q~q{>{^E{$po$JX38dN@o`-~uJo_a#}jl(cp ziJ@ce1K1SZnls{5>V6!J2_{^~S`x5nLh4Sisz$a!r@03=0}OJ!-WLCnH4362kz!AX zfWhlstMQaeY=#37e?blzPdUQ2CmL-)$K(3NqSdtPAZ}yUM=epUi5L&|7iQeI!Msw4 z*Dz%tpQxAp@Z?y_iSDJEsE_Re83Q@@1+H4l8-D-LE8MfRp%yLZdW%IiSX$z$wa%G` z5-qg>e&!AE&g5el@xU(Us=VXmeY8l&S;8})OWKMhasECjvPnOHRk#;9I^2ULY8(i* zTAgTNTo`~oYkp<^N@lO0pVV$CV z#>=)2+Z{qGGk!_L%4qFqv1^`2%9bR%K>Ug|6zK+xec5>jW-HWf9wl6hsJ5GJ@YO?$ zQPzLngtgrX+!}qOk$xB~ZuzWjs*&vN+-ALRTxgHN#$595E7wKhZg*4f+Z7Vq7+HhF zJotje0E!)7TMv97XVGXbM5zQ; zWMK^`vsK^OHQazS7EqO023)3yET)@R;f0A>KC_*yU5#22^osfI^`lz525rEN#0|Qs z09I?(=s91YXAnAR+q{tk6?`!-n{0Ux^r8Mya;6IwPK8nMg=sjo^#e22!eU5Y>W~>} z?&rLFCl`G2T(_NqP}&Fz^dH1Ab>wYjh^y2e`#h*!u1Mgh0#`y{yNMP=je`(auwmaD zNS^np88|1oNF#{F_ugsY5K^mGX&yTDJTEN?${mLodpa!5yBwO8L?HUkx$x{E*ajzi z>clH$5e05%fzD|+3Q{0d21A@awwS%P|*VfoSZi;bs>9NFQe zWh9}>R;u42y5vP&k|zbGU#{`G#$2%qY$?2-Pe% z$H~MyFrue~O_n+1psn;l6GBsQlL5F}m(E)qN^7qamzq-&yhMU?BM+mamuEo9Q^?DPWX|fYRqY_DF-q5SbuA9TGRz7~knuUq8Je~>+gN9#z+3pYVYo3WAo#SL z(y+nBN{`$iZtj^5ms4WFK|yD-P!KWaq9pk)B5yNNDf^U1K@@SIW5p=eyVlmlS&qQ) zt}ivYGF}8cH{D8`$~;HwEJK@p1;P2l5GTE2;ed2eB)_!A)JS~(5a>e%Hevg200dIU z8mC)~E#qZXLpn&?(I~zWBf^Yq$Bl26$xEjx@%RPxU$d%)1kvLrAx=oi!-kXIhC%EU zqz+t#hv}kB{ zNJ}j84MU<5ec_T6zSDsE`W%=$(I$qs*>}V3aEV_SZxxu}Ht+u9Ip4OW;*crYcreBm zf*4{GlBem2Ii!kLyx@z~U&tzS3y2y9^P&>)r^)$-6yFVr_AZ)ZTGWChHju9;MeRR0 zBO501x*5_G#VuVtOCxhiNf*o|mu;<`-5=b?t89DJeInsF_YH=AAt#~U`RLGNzZsr2 z)OGwGd5+5=+ffa{(gxEgTO1pMiur7wT8`w@nC!hKqHHNRHmN_n0wkU+*&_-5M-tgp z&7ZZJkPW&J05)0keJ{GuOr(dtf*`bo=l5jz{Pfl|dCf*?DM=-m(Iyx zYc#kZtAU?J_*k`;>6#K)h$$Mt-_7UVr>}3Gyg`2?D?u>mq_r4NnRiX;5e5f zO|u{kwu7G-_(=f2wJ@gM8B}}H)QXm=#Ba|o$d${#I1C!X`E@_mk^qE&FMGozR#eWZ z$=i6uUvlNv%vS{6S664XR%>M+C5{ZNUMjUYw4ZRM$XkV$yvU{Ag2ZSbT#jvom9bOX z%(V7_ktW5hw8s6IO6GyJhH+-(6Kla!OH|VwE_eXBpgGzbkH;zc;&PIn@Vq^?NP>CK z#Hl_DdS-w}doJUSy(PP5hZ1u8Oktqm=a-zFa;~EE#!&D^67$T3t3K}mVUwiWY~_K^ zI&j#hCT+P@Z8T~kaNCA=>p>CIV|cRykazG#1?SZaXBWHv!ll8J-!rO(vjx_-gX3M% z@rmEkms}Aavk&_A$PL%^`sR2S!0Cbomy%XNrrq{q$VD@YHbd=Q%B6XpKqAV1bdDp9 z^NQcpKxo$7jcVRCu(`l6L%3E;!jv6}b1&3PN5>hWZ8No<1m`~Z=qbVakQ(C^fqts- zN+T&$*)w&bAJfaP(t|Yc-w}%_Ry#@RN-9Bc`L#y>>1Z_B`zjr$<8(4@VyJ9YT_dqk zO<#|p)_59)_2^pGPp7ZqUQjVjQ~wJBk-}m5#ppx`A+J>KTNZHGR6OIw6uo|OvJv9m z8i;N8qjfMv^k|j+9QZ*Q-IeF?d9A8ip)>2;c1Bd7wlhm%TLgym5Z?Dt`og^6aa*9K zS?mqLsM_*W(5UL#laYfQ438KXR@U3I?pH*V)NoF_D5nA`LC6C=4QE+*%jG=u6gCeL zHG)3?i-!_T=tNKs*9(M%Bg0QN{z+OKjaXz`J;x7EId8|2tUTm-S+J_=(- z)DGX7isfByIXE1_z7 zqQ5!-_aDHMA1QixP|97jKbE^_QtqP3a&KHt4&;My>hYW+B(^m)3cWTFYvGh_5Sxj* zv4$0muz?T!=QrD6F1=H&<+7@aVyi8#Y~|m;Cb#Dm&HWh{88J^r*aQ87Lll>+1!*}y zdHyUXI8*TMO8BB1Ds@ZH>N=a74bQVmJ*%(rEA_@cQUjV0JsVkJ)2_`0xgKyPxRjv} zm@Se0U1>T)aAqx&^WM3(kNXSCXjzwGM3FVNX=FEARg4(93KA1kJP*P`OlBB8Abmuv zj|EjSkWw;%K~@_ z0KuFA$mjFI`6XtV9+*!vjVLT%H#?Ps^GYp=xT2Emns;ZyWILTk?fGrL5@%o_3-V|` zeX9hvUXEFmodS%81YKf5c`c3InKFHDAP6?hgoN(H^HD0Y<#-57ESGvEv7{LYC85VE zyfoB|!TX`pAV3j3Grgg)4(y1z)3)98sE2p#wsJe;vZ z|GAn~f5WZ7MhLZjF3VS+w$ne|NPIr|M~EzKYV#M z`;X)Ks($_-dHS~GqmyyDm!vJY|qZDj*Y@(=iA{8MV2# zTd@w)5$G*aJ4)c7Kuuc|@oOhuc?ay2t4K=Jg9FcNH^6=F$%KXEb_BwNnh_hvdM%n3I3wM5!17Roi@yHUKAxLI^U&af_)~)s(J_iV`7yk+;Em`_ne~c* zd(XtDw6hfY$c^0)d^6~@0oIP~-_Wt6a4st5Wi#DCIowQfN!HE4bv9dt58O7)WI^T% z&B&56IE!{bHM1nSnWW}$%*L3f2I7+ruzjz)K8$7MXYJmQf=-sboq!?Ccw=;D9z0p1 zMGt~mUAO-YKL)|x3C zJ+I%Aa>i@1SbKQfP^PF>nPa?iY98U57kF{g7<*5o1<)KxoR^fHU@5QQA?ix-(tD$H zk`0F`DBPXlvdOiuG*oExCT<+MmR?x}6Ka8NqP{)mZA>HEidWf~FR|zCyjJg#`l|oQ z+?IjVc9?>`&h^$`RTVP98StQbMH|`I#wZ$59p^#P!k%IEBzXMJn!lUHd$COh-qzA} za-cd+DoDMHqEx+JnTOR9R?}TTVLCm_K-`cOcG0^JA1xuQ?QXkVux=gNX z#-T3AoF!cdcrFJl8$}*91f#DO0HUwaG+&ad*ZS`hU39i$3cro0=td&e!y7jF8XdFT zOV!}T8LPr-tgZ25f@?$+_^h)0#%h)p1@M7!e?#BEvqLhLdm7ew0~+}2&eeX}Q;Eu> zcC|3D6``8id(fI_nuWGmSAy44n}M|#c8F?Gc*(VsFgtlw=Obq!uoYW@l8A(IBj>zG0P=lv5r;LZx4Ma`yYav-i}R^s`EMNPI#{O zJlK@LGWn+l1TeC-BoZwN&I^6Bx*V84bR2z2{)5*XHXs6)JLv9}W5#^F4-(BVo)hwT zpG@}h?5k{#;K|m#-H?g7a6bScR$`EhI)r)*6iq ztBeqAXtZ(1xecm{9%v4X;EN93lIzsxH>oT^-l7n3M>!jLbfauB8V2xpP(SK*CKg^T zFvW=BnW|olK7yHqS6#S|;&2;0K0D>QQ7kR13@sI8j+%Adq zqp_Ei-am!(qj3Z76Qqzn7&n63))_R7kJk>mRi2!J&IA^#B$T%yJI*BCdzg+BTvU`f zC6D0>5oYMjrps>(pp>=N^SzwrJX%#e56GtX^tpzix|8gr(8m8ObZ&!s!SoW8Ec`!B z9x}w|^FKAkng8bq8k+y7nwmOZ*tpycqGt)^N-9#ir!mh>NSj!hrwRRYSHM8NU8Mjb zZ8k?Axiys(UZ7uo894sGagghGW*hs-nm`8&v(?dXR;F80J6*zQwYGTDDpI7IkNcq^#8H zh&CA=+K3oVw7HZwt(?K#Fh>WVnIdHnlw`f;JJ^VTJ;=!vJ=}j3d^0Mep!!a%iL_dJ z+{&68Dv~zl)!mR*_sQvzKKWr*T^!1PJY>#?w~O^NZQ%c;Z79*eK#kUugVSa$lqq^a z7fVotHIbT@P!o{~&K10bNVO!Qyh1dzhz1(DflEf`^pKIQ#9n<6Cwo71TCYv&sn8D7 zSPQ_xy(;($?$`$ou+6LL)YtKn_bW9&LNa1=!-T#^D2N&LWG0TWG*vX%*NxmQ@3J*| z_2{pWt+5q883#|g2*|x40utgM_gvg$J8H5oW-^MHR79$00+-~9i;MoJskC@nV30sv zOdl=^K-7!CNsN~jXYd3d;FOR%!{JU-AwC(h>zJ_v0!swMrJB`H5U!{t2M+LIMbF?v zpO;62>ju%GV)N{u8I(OYaKDj|t9L@cn%w1|G@bZeQt-Q^;CD&E-zzEjk9=lnvnNxQ zDujV`JwbJbTeMB4ng&@Ci@nvxZbl&$-(vB z!S%!O^?`jb7Pa$Q2dsR8T#D=HqYvJ^OS=6L**L!dW*^!dU0+JNzIyh4KW_zr_f>Ui zQ_i)pA2-;v&-VKNQv>Nwu%>=m5$kwbF0Fa&3`V@v8;5s6mp&o3J>RS+lHrqbf?l={fNG)jz96z9 znR?)EbnhE8TY*kgwBy&7oN?hWAj*W?*N3@-q!ZR{#2hV&RJg4AdkcI5w}<4p$nIcY zyGrc=-?7Jc?C~9Y{JwAISUW;5t=E*+FajoJm`&mWFXRR<+kmo65^r&N57vQ7Sra*W zy!T~92LNDv`rG4nn!izo&240&OYBtc$-Y-J-Nm&n=t}UJ3tQO;*i_sWtW*7L2J9_3 z&g(sVVtqz3tS^Md2IL4W79@C2z2epy=z)UGo>lev{cJAJ)lzV!3X*%#OL2vZ>w@F_9A6TF7vy|0me&8>&Qh`Qq9;#_6bc`Y^FGKbH7uk-kOh&-Aewn^TD+fC>9 zd50Mmr+b5Kw7=b<(}zQ+-=2PNICOLjB5+OGgeHd8WNvq9uWVjRNJ9q!-x253!!sov ztEwY)eP&=kijYu88o992dK~(UEy}-9EHQ-TZi9cf8|=iqyDi>wi+^8?!{+B|?^uRA zmf^#)4C0Q=-1@2Dk7GUz9~p8%J=SFS#FxBi!h^aI_d}PAL97GlvRHYSO~^A*@qE{2 zGgzVH6G+-+9g*j}hU5>aSud(JTS#Q|MRv-3%i z?Yhs9+`C41etdU^$~!~lZ^}?Ma83Qt_S&#-*3re}WHfQ5=)R#_#XK=-mCY06IVp&F1wM}J z8tI>0(0`aHFL+9|>EUBt9kEx3vE~A~w@Y|-K`WC)g$PCGdaT5tsY_5vSj?05`EG|0 zLp-srh~djv4mMk~yR|#tIE@ET|G7l5J153})>xE}D-L8_kRv8jNKs6v&EZC+raF2( zju=Fc*lK;Agg5ZOhRi>n9PD>^O#iWu$GmtX6L>@gXmZZv9Y=Y`QQmQscO2!fz)^N- z(H#qN$Aa9kARmSWv1~>!3u5^Wrozqw&y0Ww~RLpW0l>8!h&_!DnQi*JJki0ApjlF8WCJdBf!I?b~-#LjZ+9dcO|e zf4sr|4GZ7S^J%OhHeaU8v=1xpo_CwAP{DETp2&n8^pq(cW&*>^%)@O4JR9mBhmI56 zL;gq;7U-YDchAHb;3f)8k0HqlN|;JAB9vPv-APA484g?7tEBt5DOZna6_{)ap5bE2 zrE-NCentyoleLCM+pujC$Edf6Zg!0 z*v>GE-!B!pAyp50Sn~+N$6{!Ho$EK4?|Qao%~kxNFXL^l`jF;rz#~8d=*qwTVN^yQRqQ*%PzN|5{g0=%ZeL`# z*$m1SXOQ;WKZeNaxBV=K{*q&HMcG=GWH?06xX6eZ?-7>E8OI_=Pt1#}x&Po_Xh-Cd zFhu7l94>0m9xbuPL&AJ6ey0JO_jDw~Me8zm0V?Y562Fuoi?^=QCPuH~c_eZIy9+nK zy$HrVLnZZIIr`i@7w+DW^~u(x!YntdRhFC~NL#0`vnKgk^U%Cofsn0k#O7!e>ljdY z6Zp_SAf2Ej$#`Kz?6gS`pTaB0iBUf$T#{ZwPQoI}R`xx7{aXJjschOBx2 zJ<@r+n=H_yB)O9k?xch}DdE?a629YCsNigYYNlwRyEikiFE5(6xzP3k#Ahx){Btt) zH#|+T1(yk&Dcr4^ub1a+F3<04_DEe@UX3%jOPJL6#)I?pkGJAEoBXtQ{Qqae|LfBN zY3QOi6*ajzEQnApMV zsM&$UEiAemr~8lX4WL(03kww(wq8F!Lkuf;j5l>qm6HK_tl9{!2~^fnSq|TMpMqRr zbS?f=@dC3&%Ee+-T`WeKSXGLTR3S&h>SB>(oN4jECSF#Bd_mdw2HF(af4qKvHgPGq z-bf-Ox>|B@x;3j|5(ZOgQaI}y@Y}yUMU|kJ8dGdAn>E@oV$wk4JV$dX$d#5gjU1x5 zeeKKugSCNcHZQ3(x9Am_Ejhn%Z^0`f7%q$(!6W;uD#%Q>fvfMAT7zpg=M=j2ulVaR zk@f}d_ivo;kDU{R-4g_nctBYZWf>PjO9Lz2ONo`TBBq@kY)XkQY<6WT&=p?wMf0JG zE5Z~7hd2w~rDSd%KQj6>g@rN%)i2Ut?Gkwj@_S%Q;C#p_dWLDSi;fqJ3*!EM#f~)+ z6oqDET(h1clxV)<)QEiPv9v|{Sk{6`G|(tI*wtJI8kqJ2#MP3BD=G;Z=)R59;vCbW z7UYB%G+TwPjNy^CtjU>qx67Pu5_Q^kMzqO8Yp%Y}yj2{jq`7W{#qbZ?J zuHYQ}+5uei`nL0BH$XD{4Gbw%6`ZB#aW$Ygyh7=IC-VcS?Eq+)Ls78=t$my zZN7B@x~ruOH2#B3VI?z8D)oIN#D}kz#H@slXvD1 z%!;q)5o}-YcjZU>qr#`_yL9U#vjE$$0cHr{@;aho!dvtz?;VWwRmw$f9FDKJxX`j` z!sm%vl4Qo;uH_W;g5gU}b1>+$MT=8N{UBl%06Gnigi(=cp^cfi3&d&n+wT8;RV-R0wxaA4*Pft#1-0y#qjXG;)_!fL>Zz@<#* zJ{yEuRdBrTz2rI3fbIoNn)R@SQ?x%BKN|0J3SR5M?oa#^{ix$~qA?r@4zI{<*KaGN z$~enNrR2!Am5F_FhO&kGHX4?7p=hE9N~@8YY8Ef1Yz9!E;VieO@bq$Xi1O zkP>)4r5?nWUwEJAzJWVfuMH^2qCPPIqVc)$=v4{7I~3e?t&UH2 zQMh7#{c0f7ra`<*R4RS#9iQk86+}ucttiWwT^2`R z?Lg1@0zCtv*0+2k2`czvUMh6oG64_tLHkj1rVAE>oS2^ zwj^RyQr$htlOC;P;$`u~4j^Efs8`G*Tve8{LUT$A_lN4dlib*<{6a762P{*tQ`L8M0c-_ouI=Z{_zMx-=vOt$y1u%#!u@Fu3&Dji-g z1(8c$P@7{K2m8Yeq7xyRZ+*BvpYK@x9Gf05gM|2^=?SW}-o%2TjcHX@20z@zJS~m)c5C z392OB+A@#r)n9U{UK-Z2tA?;N!f+Uxp^=8q(3c1nO%jP5t+%Xg<|=?eGm#gRvyM!p zi@S;xBiHl7nX&vv*;m$cg0%D%pxXBQcBBv+0hk1{JW;;8&LEqjwhifmtfpviK~{qR z>)Z`7MT4h5)wl@DKtVA@1Nb{EAoDrN)D#U~a+erm(7p66%4wdJX-ex=>JUHEl~CS$ zeK7P3BClTd&eFJ%v@}1u?x8CRz9augoc8>@4y zdD5)PD^{%ZfM zg?J6JvDcMRwm^p`$0!pdH+D4HQL%1TtToPXU^Q9q#o8l8Y!DSFTWHdgumxpr6R@A# zs~R##E(mLJ`g`NOy~*C*IDBjFa*`gTljPu0viR!JR|J3caQ5ZxWdS$Z9`*|)DKNue zFCL;y%ZJv5i$_)@SRuXSKk`{eSnZWA?GE%R47aw_yoasEj;$+5n}QDCVaMCO&f1rM zvx2Q7g*5jpb(cw@#hiu&V5J0|)yh4AZ2WT@E3YI<<8y$(==G9%A1-MWT?}lA?pUZL zs%ftIVqi1%v%r~Lz`{_uj^ZCXzn?ObR@per)tmGN}|E1U5+bJpcPGn#YFXnYy& z14f{NO7$S!X~-9qsH{~D9xmXHFX^Qf;c~)O%{WqYW+NpV^a8SWS4<1q@3Lz5^^C(M+Ci;FaF@$) zh@R0x5fMyBcUfqxZiS_I6{wX9{Z;R}z=2mF`0=oew7QHd84;vYrRj4m6t%>RgR|AT zo*H*PD~23_6k?6`-uD{Zvn>B%E2b48!U`U4mBUe6totifOU)+&7I}vp*Dituy)E&j zOI8*_OKkWAWs4;GK2D>bX7x!veLQ~r&}&i+zz5BUKJ{=?k2gIcAB@#!dXG0^iUw{_ zU{M2tVXcyuzch!$C&7Tb(eBC1uwsLWk=st6JB-Hn0$+0BHrDot3||+S>xT$E0pC2I ztK*AMVW&?=V?|9b9sxIVIT=M`Yno=XpmhmE6IcNF2~GA!PvA$l;T*;qY7gA4WSTcw zL(vluUyZyOI^GE^6bH5w$F_D} zXpHXNY_NTDN6Sf} z7lYe67r2~6{VoMJyAymeR{}@OZge5&IuC5=HsI26^*IZCXh(q$<|NQsmcTh+3&()i z;oiV1+H`OWqk7kIzsKl*<7kXY9-4Oy{M(HqhR|cozoj96gAsp?sdsBL?-~>DEzG;U z274Q~dlr&qqvz9i9OW#Jl@bvibo99ob=CJPgVuZeZ_3e`>8tjkF<@Ze_3-KO?&*5=w7q(^_UO66 zo2TW;)9=O82+O@)kq5KW8Kam3s4{|0Ek5qB$LgGABWmq#^?H1~C8dgxBUZ845rqfQNL}Y)pok zV*pZg6nKebRc-Vq!1Ne&a9&0EY2ygw`N=bDmXh}Vc7w2FE?C|M&J>s(&2xcZ!iE2R z$Z`-aoYg02^u5hY(5dQ{!LkuH>fam%k_W5ekif;ul=$tTsT9!Dm|q;WxW8^slx@=! zWw-4KKGs!^6$wpe@sgL6(GnMdl@M=CE?4rT332)Hb7`329( z6zPI)sur(vbm2lkmje&KTL%L5lg6Gvp3re`QvgpiU^6uLm{UxOH4ZhLf1iqP7}U0e z;u|cbrvEte{a!!SbgJKgca8hq;EYw;E2{O zNr1|R`$z;SIL;fxQ8N?6Zl2n5gWi!Z>-*q(xA$J8Z_wTnN}T7RSYp5+2gV0`!QmEh z_>{lwLWamN8YX}$W;Xm~X8VXSHeMWi^!p-})Sr?)W~=bPg#iI}Nv|B6V}{=;;#)7mqzVk(4<<{mi_Or!nWI4WIf(Tn7r z3`hRw;UX}|b$y8%LH@QqMkjTH&(pj25V$qjg z|NZNZ0CGV@Hec-am9`u<62&hFvo)8bC{wu{S*N)4a^lAPf&A~y4s7e_5%@b?@}e#q zR|7~c{Jr@Wco}pWAd>;Hr`ZcAFm0Qh*f_e^>=wpJ^42BCd(9V%BGRZpS}&||8U$8IXcsq5C9Vq()VFvzQY}7cArv66o6)L(P`!--hi9b%UX$JoaJPY@T->Kpk%| z&zgsTdVJLWxjfCRd&*hoT|MW6=6>7Y6z7!IDo+~e##ii=R6OtY)BlVXf;Y4}oaJVB zHlw@zto_(}oXfT(d4BE&kIfD5oRp}{_$3i5#2H-s85W_I#^hea43+qTIQHN_H0f|D z)}ZzeNdj5Nzs{MI;_{E@3sGh1xyq{a4?l1AHu%MH27V~%AAa^0v0p-cqwx^^L2s3> z4BZE8i{@H53kjxqZ9rGMe*zbfrgQ{^U;mmqx^a}iTtoxYL>K*7ONBCCtqk*lr6T|R zfxolQdY`UrxX2qk1|0$U%skBce9UfiA6uNBJTWP}f|R*UZh4I_{?imq#*fDn%Op0> zc@A`~#<2U{>(?hCNOWH#xJ^)Rlxq42pF|1lqtB+Jy`ZD-e91spYC7{X0`&(NBpS^Q zsQgS20h|-qC5)B?=Y^Id+-bW%c*z+Vn6ZVvpzyh|d*>PtTd>>xpb9If{Qy<1eM2vOEj15HrJ@{_ub-be=L6GK zFm4ZGYTmw%yoy2tvRkV{2FEm*i(B6r z!GZ|5X`X>e#Es{@y)F9PEFAcEhud~-=-iB+cw)#+0ub95Vv~*-E^8UM3Wh0$;Xd=X zQLW(dsW7ypeKelhzTfqxK=0|p{9C8azr7nWa=zYz@V_ftEzgchi zEz#>%2{8MjO@i&~%l~-$+>gJb$c%U5o41Y53A2H2h?_wujzBdfUf?z&C&#r~+83yu zTwOKC3Pm1!C+&h5O2av};Own~v)Az4oaviW7~3>1ie}}(PM6tolB0njIA0C?Q*GWE zOf5+%NpVTMkJ+|IPLMefgk>ZsOmIcLuG1^-{g=zhXlOV1tRnWxtY-z4%a>f8>LDIF zYSGpb9owwqm?xul*g`Yyg z=@~))Hb(#YujtQTKl<|FcnnO#1m$@3KyRy4bU0vNe~kvNr@@9{TlN)-kX^3hlO`g* zo7w--c)a%hG!dw50o)%rxRRKY#O7r@=fxYIQ)_H*UAQMiSd;nI9N;#WZ3QNBfcvQ~ z-U!e;UO3Ha-0JF?^j|&J9Cv(z1Y8lf?D>W}>Gyj1MEdZG=P37S_t6aU?)NI!x73_& z&hM$XmRH?Lg7Ub@C7wD#L+iabz#!_ zIKLO86tr)w^a$3?yuyD%MhrQ6c(DKYQ4ojMd9PsI32(nnfQ-LU^qPS+_n~8>Dz_&~ zdbRF)nX@t2hFbQ1hq-55puaSRhkyGTP14CDAobQa%kGfPMq!dqHL0Y~CB;x=lF!-- zK8Vovaz@A+OqmsYG1?pNjYs2$qw(H|mX+X_;QcQX#UaVR>lIftp)3jRXg$^lhjevF zGnMdpf|+Y-htVG06hRiPoo!_xM>+!0_|q}mVQ=4YsieO~?TaS>*I^5lZo{LfkEL$d zN-}Ac7z?b(6<(#*(>LQ~GdZl5GSsNqg)|1n=!(%bqj-{0V;s8`s%c2C_2S&mi6Wwe zD77WO@-u|40tdp+>9tqr}ZF`(q=D9^c8A8!6UJD4ctto=(wy3;x%yA zOD=0J@%6*U{Rivc#5SCaj zNseY%(hO-?By<~i;OzEh3_c8%22p4b*yp3<>LB5h4adFNLZ?S>e+-SmZaGEQ|37>0 zy57c(C5rZMJp~T!*_JZmO&2@Xp7eLrwv%YuPSlp%>6uPODY8JaqGF*^1xi-i+TVGN z^Lpn=&WD9N3UwnXS$0@IY_SShShy`Ltm~FkRZa*wdK{$L@bQn|ez*URn^F3BeEE9t z?e{N_FUddt>m>gEd2f7^pAS@xoXjyvFphI9YQB8)-~ZVA?}OhTKRp}$?PPMZc>TAq z_kK?Lr-S+4xW~pj+EDzpQG9)S45JK>siJy3fRw&j3n$u;%Rq$(t#wPcPYz3B#Yj2q zQYGhq)1It9DsE1#3UD{3J_DN3chm*rbVkPH3Yn`sV7eJb&U(P-o@Q7%fozV2J31DO zJK!B14aLL3P%m-%7^?%Ka71X&DPtuH$}O3Tnk#prCRKYe&w~`$EXG_F@7i|28l&OD zrB4ly*iOn&o3~hVOW%;viEercFspmnG#i z&d&^@ZN?lCB1~x;?B8@^j%;fQN~YWohr5yEjCQqMw0V5+rd|4378BJ+5~^@>2!bRD zo_2d_B|%Wt#UH8Mhx*Xy6og|2%O`wW0ZZu#n37QYei{a)Z3Sn_H=4ufrzGquk5BQA z3!J2vr)ioh{LNmNZklpWi*JnQJhe9am2C>Wp2$76l)fxr&%H+7S>DEI6R&3zujdaE zuV-UUx2i|Or?;)!L^ArUA{n`d(<`6qPoGcfv(;g)-z_o52LM)s3o=jT#Az0m&D!b78CZS2&|CfP2Tg^?ADgZ6 zWgxMTaLiwVY^;j1G6-I~|g|$1R%y_|>p?YBQG_|KrRt zA3l-1;Rw@f%)MH32>Ww|1(T#ju4asj{egbJmTn4X$nKzbZ~&>(9k=-LWZIjON3Vxp z54x|1U+-HSAs~99S=s$Tu8&Hc%0B1jERZ5S5NZ-moOo9IJS9me$tc87m8txc1lbL% zXSGy0nwYUJT=-*$V^{CXI`A6IR9=Ji!#uc@p}h+Bl**h1#e}4zV0^iYa>AJ<#uO8g zI$;zgAqu8Jl;l(zhM|*FoBh#kx`tWY6Y4meN6GQ&iMX=nCO(iEfPIg0nnCJ)&Ly$p z+#obc#_=M=IhrhzF<+bPIw*s!f%O3}Ezum51d*`}C20~QCp6-*`j@0Z*W~5VSX^;H zaZdZ-{uw1xwz>L`7@>sRvPqW_3TwWs?z@9?#%>dW!(>a&j2C`|vn&cFFIMD8dC#-x zO6t=danm*si}J7w!L*0o<&1@3Ozk+xdGrurzCnYjp5+OFgdjPdv$b)x$HzuAW$}_f z;nDf)GfoH9)k%tE*eO595G{{w7dDS)_e9^W@yO;4ukpyD)34q`1m7$N?uJWj#97TP zBr}XM9*#?^s-w$OQchX9ST9#X=mL z@Hb-r@Try@g3?ONgZF3Azc7T-84TpLM{NFYrAM_%0JNKkDT|PsR61enkWFoOs04K( zLG_LOC^52FJvrWz|IvgRc9XR|ZK=pI;W-?Uj>;_8I~??6)E_$vM~n2TcadCGN;W)+ zRUe^ZURBUk`O~VxCr?&Ycwe4Wl`n6Vlx0`OdwRlFQA%Pmy+L`(7*IG#X$IrrC-PMT za!s;Jw)^vbO87rr3BSwXcMbT?;P7&1Q0#>+Wi2$eSuXLlG*WwHRQxL&MERpmt-nk1fJXTw;>K++$q$6eF;fNM9#{1KF7^NRdVmW)*9?@yxFR^5Ap}CM8o0KxHK1N5+U-S=DMkXF8wii-xhr~>&jLO=5KII3 zR6$aWDy-ASoeZls*+5to`i1L?ZeO0GU_x<*A|7VYN^BMsv332IGCX0! zqZPi{A_46(V2aLFYzwVE&1-wOYlJ*i8fIx6&x4~0HM2zHnrEBCD8biiK16f29*e)T z~$*kolaAZOk?s zI}*f+1Q9P`Mhk5FU2`UYST23oMAzV@wHeuMEsA$VMma>T6Fo;gC+R1!R$|543h=jJ zP^ja-@UvhM>s*e7tjR$0xH?BnIF!f>P@}|5X$gy!Q)N@9&#s#c#ZFG4lnG*^wWj_+5_k z?{Per;drcc1!uQoCdAct5M|SO;1?e)Wfhi(uCj;bmvpP?F3HBJrK zH3O;EtIN~i$_r|Fc2*(dj-f_#8GcrSU2!i~q4BBqVMay|m!k#j{JYP@+q3a3Q`S=s zW@WZ@&!;2J@@2m^wVuY6CHOn&2LL=NDK6;JZB{;s}0}xs{5zK*83#+rPk~2q3dRq-gts1GUU&r zIsPe1LUMhC2DxKvg~ntqmX+IwGj0EhhlmqG7*uXWdy_)!UOduYNc`etl$`6Y?f_^lh3&qXosv zql6@s+$FX=#Q~T(<>LSXv4)I#2}mkWL5BJ8Lfi<36w2^ixZO#0TfU1TkN@19Tp8`1 zB#?)m)vIg+N8kZBT7c1^)XUQ-S;$jWh40bkZJ^Ci=x7}T5gLJMOh$kcJG!0d=0Ot5 zSFCWv#w?TB8(^Do#3(@l8smg!L5yO;BQ2?>&R7q)69hMQ7Cu9eJ20Vg+x>cRA>vzJ zT>L6e(mAF<7YL&*e1KsxSJG=TMKQj@vFqJ9!bUzB6j=eubXxx?nGg#WWU(tmJF5Zq zEmS&Ue@^DqV9UW1P%3A6VkMmMpj`Q`X})NvNBL(pio!Ngux9Tg>Cly7J`wNUzP6QM zMks_M!;$uk21&@46xU)s$zhDyH1^ON?&L5bUVV=de@YlI&>N}TVg2cYV>#QaB+L6D z8`ypxO}jxho<$Uo=_14ZAdR|Xl3ek#CkJC|(Y`vo#BZ5OpRGbO(FPEKG$Uy+4XE~N zHh9g{(6*_2v4CnAeZ>BgP_m^Mi-7-U;Qxc*ep;1@mryU0gN;AxQC^%8Bxz6~W zOygi|E7}aQZ7Lu|-V>!Uw9FA1GcutaG{-?=BvjOG3D@SYQiv`26_4~rRRKh=iL#Ag z!^H`h)jANGgrMmn$by7oh!Z7)f>@{tl-Q=MKq<=6S28{BQV99l>~K0LEEstrNozW! znRY^)4?MPrv{umDT1irl6B_AYl(u1p4N6BvVZ5R|X_S?kl|jv{Xd7OO;6x%K9HL~` z^kLgYlOT#08J4MVN0d^cPDo7uHXwZE9c&b=-6UMOdBlUre8zAKj92n`6Rz2QwWQ44mcQ7(|iD z3*FVi(w1>~UpgqvZTXl+&=jh?gKNsE!j>nadl3iG+^qdbhID{0=in?ru|d*u?NrG} zOSET%_nE3_qN)&ggm^LcG%#w%(!tqo^n=*Y*O@}`oCc&cmc~1rjv0p2a%F{nvF_!F- zI-H|SmTNRAg+o2`pJe1#1knM6cY-)wB%7;Nb3?)=F-p5S11I zk0AQYjcl3{G|F&ri5BTJ3qqVbPDi}-iINcU%RbL_x2uCh(iy(d3l$=t^uC(swD;An3b9~nhD5yLz5Qp; z;f49%T9pj9O7l8={XH>vx2?bsl zkS_6!L}(dE-AjC9T*P47i{qvve?WnGkQx|>NEgL$84xOiL)`BD`;GyPAN3cp4z-~q zi6&Fv#=X#AtT-A*?IK#+|NNh$|NYCq?;LIa92z*d!8Se&Z;#m0u0+z?K|2-*^2bO~ z=!h-0zGiC1##-lkP$hqPjA~1tZxRPBD#mpa|}- zvI~I@*&!$MpDc205O+hBAov7EO}Ri>NRJo*c39*XLnK5{3W3kfuhTI$$QzHGB$Ox^ zv3^M+R!UAk%8r}lv;#gj>{QOE%x3{(W4C8e!w(-ao+6bsijKZPJ+6lDO~)w=aDJ=T z5|8Zq8+U*ee+@e(ThD47q0Wa70PptJ`Y4i-3Y{ykI)N< zk?N#d@lR;g*(su!>x;yrlF*9$uAqKh;u{@a%E$_(ZAj1Ti=8^rolkL5^!) zo(;eg7=b6w_*M1Mj*g?l$;3FLZ@7c4bA^H335rCAVIvdUFZVYxQU){--_pD>$R?X! z9M*VY_zx3Hv9XJuRL_?RU5BKDpST@jAwNEGUY_^QX&+tJsDCK>m8gYy&#W1LV1A;eb}k2Ae6 z!md5pePTirSrA=uzc8vX167zSyX17Rw&-$a#4fws*%{mzICayV;kn8d7T;)70N0A{ z6C5+1*{@$8o+HO|w#XkmGHx-rl0#fYOu};ckh!*51t(CTmJJ)+*UIt8(>>KyHp~)r0SW7GJgeEs5#?~lEjP8Ia+dIO^&{H0g zQcwDY&Nzq~1)$V-qY7GCw#5G+#L}LX`gvs$cq)Q z!-D38SP>#NB5XMU6DL~Wz8BDjC{YJ>ovm-Per{;SVdC<<4qvB?Kv|e+$92YE#)gj&9zj> z5*bf0O2lH+0ryA|fZ6XyK+^I1D2E6b^4N$bhEdqA4@gf?c4e(A=(qt}LiJE(gEkw% z4Gkh~zkMWgiY6h>eAviy223Y8jh1;MA}Z{I;glKOC=$S}yX4&NTFph=T#AcL>=?;C zz7%`DF|m7$VDD24u-rESNJRQHO2se37@_|t-#6;zWh2N5=h?pVdR&-i{_kipzGQF; zLylAockf3oSN4A5RtXoyIWsX=p9O^`@70~fiiubg6)}!RS&-d0x_XF*IPC2m^akA^ zP9r04wo0O`?l@8Mn+UdTuv+RQHzHKev0$0FZjP1aX|aL@o#BLvl)55iGZ=xNT1L2v zUaIic-t5kcGz^q|KO`~*?tqsa7n4A1T#t|wB#CIljTTu& zOE+5xceFG{blYkwit2lBPr=-nI(v)W$qppxgaj4YcSkcg5?oN#Eor$KoyTq)C! z+?NA?PZE*|2L?#D%wuO}3oP9|Fsu`P8#0_qW~`!6-$E`T3lf~BjK{H~XQ)nF!2v{r zyZpMUn>CwYg`fw#r=*B}We?AzInHS?PfcZwaw)>egk)pP8uV{I3v@>Yp$q#bc3I5Pu*u^!5X*Ku}y#V0yc z1!3vP`Og*WV04|!bjSpeESg4%PPfK|K5%K{NoaVK{aox6$5-!sCP8tB5aA1c9o&Go zmUpE$K77L7uG$$7?Eq7+-a|zG@Y{^c#XzC1ia%&tK^sBGu&BgW zDQ^@sk`p+^eWqJ+q zdpf%=u|wlmT;4K0RNCdIwWDPln)_iIBPymB9G`F^Jj~0t*sj=LVyBsr<;0#h zr1cduQ?a<0pBgQv!*?ie;Qdqs#@D%joQshrUz6##aL5XkLaxtU75AEr>)&1rCsHmM zjmdP1EonTQQqp@=`OIEt5oH_EB78!>FOB9yal2ql0i}!$lsr%m9mn8NNuw(q-*9z- zYD0h#10JB9hGaqY_k1zdeiJ$Bhxn>5%1{En;k-xSGnvg*iZGMp8r%k}kkbqYbIta_ zTF`F0QKFKlX_br#bX4T{)%z>G9zhFGKSfK_~*E&P`2H9_f+K;mXY_*u3q6F+c0i7ULN*gyi%0Ij@)JBgP*(O zKo}JzBd&^`O|`btP~;necNC2n=uu z1uzWfG{a^>jg;h6Tp_xe*zSsKoixgn--8-#5tAy&!PlA%O@Ug~CqI?#`*WbmfqQLa zm34wAt~Iml$pi(09fdlu+LEcH;KpcvM&njUb-Tm}HcxkJ458X*?ie}*;hx7(wqBEe z*BZlO0$#^lDvrtZk5@R0gEWnj>0e?pzRXohsw%6I}Rg}$IAvtl>#6l~(NXKN(7${3U8+Z|SyZ0taoD^a*g&~`Kk*$Xfo{{Yy z@~iW>8a%&6sP)Gpwsu5Kq2f3T@|kpZhLc1#&m&IcQHa?Qq4@uvC5IWHZPug_qM}d% zY`F+ea3`Fk7+O1gDCSY}mi;{AUDK=-dMjcisMJ`D;{?N)&@`$Za~IqkZOi_B{uIf(0e)b~z5>SX>DV=*}gDRXUENAQ#3wtQ&K_6;$@jlT#Q> z=~m+$!(4A6trWqHFkxgXyOHw5l!X7^|MUOm03e6EzQzjL#!kYs2oxs0;h@LG}e z3YK+k7X`y!kfuF$6_4#NBlC+A17;9ar)V8O`3Hvt-H%vBR}om4$+%5~OPm1QL|i<- zC(l6%!ewnCnq~|s3c{5UxxziPod!ABA97U%qG`-;_F5NpXN^l}JZ9Cd+%E}cq$|7% zk}*S!t42k%GGgjpt9t zuer%3L?mJdbBrlxU?9mQRYC-Hib-JBwf^Y6&B1)UpVgnwTIF(X+|2Lp9- zZVwM7ym^!?D9+_M#7FOpcj={AQ}(H3gcw=GIRBPpf*QUB2U{Q}KuUr%0u3yqFsRc( z;)4_NL6Hma3GPc%0C;_cvk|X7#k8}FI&qZ4|JMN>&lp4tA16?~#Klev=3GH35nO{z zCvK&;^XxXQjbo<*Tux;k>@#l$fF3$0Q?51TF?3~IHvwsqk-4FpVysg}O}(w@h2_aC zCt4MZufZm;4K*LdT;%;58S#6C7zVH}{||#%{&V*~Qj*a>D^2+X_*p0rZAOFeYO}CX)+y0vMSu{E3;N>(A`+P;<&so^g^k%CDN6#k zpSX~ivar=N)DRF`)8o&mshd+7-Bk`j;ZJ2;v4mLA2Ru?4 z-S=fsUjPf)(9?j8Hj#I1@R&3rAUmK^ZbBqWIIy@cnP~?j`|g8fd7f!V=daHkeoDY| z2Gf{kA9S6$RKgbUPeq3rfEUNWkXr@JA7~u7)8Rnjp&THn1Q<$FTn74nRU z;S09*Z(vb5zGjpF#fB@NR7!9zPOgMqh`lM;L>^{4D$9F$7vbgrUE{W*$m87XPn0-r zLf%*v!cFK2J0vf*xnX6F=|}Bb^D!miM@9KRGLG$IE`mShnC|rcUAtp)7Vou=Zo%qZ zT>N4LgR32DX%=Vh<4-!OS;=@7I_rk=NJ;1m=TYUZ^*`IauXY}*w}t2)R;bE42QC>c z#NwBYL_2<7*)~yI8=rhy$kB7xJn|_C@0UMLNvIBp)=aX8y_DRl!CD%V)10k5wYgfw zBIk0Xg~rba*~G?9h4|&Guvfz>o3|5>1~{$MH#O4&JAE~^or4-CI#sEPzf=ZVt`-mX ze3w!*5(?JLVvWpY!cW=UPAb2plKz^(Te+Lb+eM53t3Xu0vn`kA+YTvXGEYf@6Pow4 zMGPJ(TaO-X832F_<92T`xv@YC{G7-ryto_LGI7V9DP5w^3P+m%mxP?)k zEmCg%y$C_0?i-?py~`F!7l_!{_CLqnG#vFGX=nd#!1}Kn=WTBxu~;!tp;h-jgqbg&}Fz<%YYRJ7uznGbb~}z-IA9Q8$01vvQ@Uak<2h) zo?PF9%0yZ>>IjDubi%`H8vq z9*Knh#Z)qq^0>YQ=1kGp4d=jM@g;0bbe9JniMuq`VJrEvQ$OuBC!)*AMdv+n%|3}k z!#P8ayc@{bKXATqjljvNZEjDp;#qj6g?%wZsz$`h*22YZ!@={xo|+Qk1Y+=eD=K-K z99%=7X2ClZAx=d&5t49=c^yLW5jW_77QsgdQ0}qy#E8I?A4rH9kSAtyiFkR0_6N@g zO5yTa@P8+#NJOv1xfZQCM!~{!UrH~dTpJYVYjpJis{@GlrS!P}NM^6;Mya|rc*Q`W#HFPM_(Xs%=KGS)HpJ(r`(;2lO64uXM;AmHKapr#1je6b{`VawWUnA z$8W(@Yb~veWx|!ouH(H3Yawx!Fu|-*trEM3OJ~WI({EJU%$l}AtJE~(9IW4hv)1^p z#9~dR)-sGGtFc?&P;wn87VHvRBHP?1aUIweEdVvTaPqy&;v@9#?Q0(SMR@r`T0F&w zDFK`mrrNH-3W~A+q>7|QyV651R1i;bApffLAHSNJk#ux{D5nuY^YhndGF2#V%4l7y zP?k~3aHpHvcG<~35p>#fuN#vLlYFVbFmZ0$5b3^1D&^|jzEDNPMh&t4G5r#h5bgp0 z%Ex}`GyYXU*{?P7Bv`QPo|!n;t1AXZjmJJi!DJ!=J{T+zkK%M+L8IVamdiG`kcr8e z&bjji=Ycet+S){rim#mqQ&7^J`*~L@L{DXa9zNdUKT!d~^IQ#j`v<+DPmADBLT`FS z`%{W9onPCz4-|pltzoRrxos+=7P*eg$SIQPhkoYMJJsZ{H`ad7*#|ua{K00gcl^+m zWuq^Vz1fkv1orN#S}ldW1Fa0Hx}5AjSh~w-nNB5NRGwVfPWMB#|YvDL=MTmR2(YB%3fNc#~fJMsh;8KKx)qoVsClVV8SlYU} zfI&~NYQ_B{WXf1H1u?V#;>6vvUY1(*K|LvS=Bi%gn>D~6Nu)BgzV9n6ea zT0){DomIvJjKLru3TFFA1m3npYU=%oPv&zbgG1)h>0e}%V-qnW*D89X=@={B8_cmR zCQ|?885)gzxRogv-HeM(5-v8*w^)*Hu`Ju7KiQ(!wB*=u3K32nB5zGmeKICVA{lUE zxs&L7xgBo<2%?CFdBy#cxMx%}ov}^*CPg#TGAkoG)8;`7<3}X5D#>m3q z$oK#*Cl^Anc7(3>jBf}Kxv+zc%9TC}@@Ra)?vd^n1i`HMtmI^lFWApQnKa1rYm$Yc z4Et%7$r-5_+j%mO!KhdpKn;`)!p)Q+KhS87$pTbb#xeft^zEw`$LFtJ_T-WN?(J(~ zWCS_EUiD`G`U=i@R-INdz<0Wb`(d3t;CY? zt)Y5lXzRNPYrW2cW<6o7o18H1LshpAQ|&!gobc4B5SkkK!&0NNkkrT@jvAGRqDG}* zsAUW{ZSyIWi(+Vbe6vjJu}zCtN>r2w5Qi%2Y|_7feOD{7&vPA zMw}ki$le$%I*5k^xe*C+xjNZy(SL$?I|CIZ>-XaI2|E^mXYY9#ORCSjq0Lz8jxV!m zjf!`?S;amz8oZ^g z4)7j2R%RAfN+00`i+<%}e*v3VX8rk9vot*XY7mAhBQ;bLdqOy&OOQgQl9KRAu9M^# zzUDG$o{*+;CX??VkA3Bd9c(#>W1%QgP6%+a$B^Soh9Qwr@MI_volyybOioBP$1405 z5Fl>JD?!^jhJdYqZ#Lojmu_GPaR~lELJ=e4R5^f*XE6!eio2?zzAe&| zVjwM)D$0O(Xv-sXFc`dvEN7}9p;0%OfOPH#!n39s$WS?_F2+@O-ls7S?PBFsf~fXJ0jMW3hklW5;xJWQizZH6*u^!ssN(bM7GO4WRzEvY8~zx z2|?3EkOc|FAc&Nv-xg{DrCE@L6(~hH`bsz$gi9ghYqP@%t_-M}7N#R7DnFh5fm@Rs zVX6e9;%Z)x6B-?hN)``#GI1bk6L>~2`SzkDB{7-a{1xAr_Ci)gI}wx!?{nM53yBvPkfbR?pFLwsTUg%QRFwKMa zXP5X|dOSTcD#G@aKT0V8LU2=gm*fGB@=3%!rCyRB2$f@|3IbnR)SQ;;P7w;yRQh6c zIrAll@FAvXL|U+${oHlipbn2T{oQ4*qH8nDAb@|@`8)Dnv;kq3m9_gyL8o$LZM_I^t8 zJW7qoi-vh8zhdb=xL2BcO)ZwVhFunxV_zC0(-KgoL=uc4ShY@)-`f%m~&o8fQ3Ui#dq%oh`)S2}zvnS!8&+ zfbc^JQ5J^UM~338^ruE5#l83e;@}0H@ob*_oS6`E^f*Yf;o~2_{cissH>33N`11AO z+wWfKKT9d)3edvP9`^t*MAFp z@8_g{I+*W`d*F93EMk=uCe28=0QmBcWo!%9Re*I2m8NJXbTn!~sbZ89%^pn!&z5hc^UOzoKWfB5nI z)me|er>nIw7z_qa4-Vk}!C>J2KRDcf`lsQ+(`Sc+!{Pqn;hzS>r-#F5e?o(Gb+YPN zup{74gO%HIJNKD9A3mVRy%)0}qdho>Zf_a6MWJ9q-vEYhMIdy6V`ppY z?@=Dnrs86!c@Z)or`bYF&}ZaYM{Pp?%g`>EHXlwlEyM1CEBhJsmISr;8IYXH{eLmT zF*}NSG>uniqk8}EKRejlclQ6YgW=(3|KG>+2)*SiVC%z&t~5X_&*|H}y@i6&L?!0s z32=JjfYKQEcujF%d@JgyYH1cFbb>nnmiIeL0l-OE_}juxIp+f{*`9Y{A@9qMDJ0j-|?3O+~XWu6`jEVx16`0dj9_RzuNLp z!WoXmKRUF|;UoB)U5jx-y9dvnJ$v@`{`2A1yPTWcen4tK#z7o+6P!~VcF;Bi;_9K! z*}K2I`2P6q`Na>%Z(em!=ls>1)7Qu6uR1&E_I3->&DH~%X2B##f{yyDH(A6nXvF}` zSAEe*IjLj(08UyLB+o%wZ$KB7F0BMK2EIz0_W9aBc>J@r+% zU>tn|?TLAkUo%MCq1$sRNqF*i4;gbOw~`nsjmCBujVbEDsoasj*vZERFvzKHZ)XQ} zI6@uOMn{~L9geT~g*q>SG#Ev3L?bB8zL~Ff9MrZRA&Bh)CTHyGiL>0A5iH({%aJn;mv1>v#!;OD8PsUnuW}H^Q>BL03vE znK;%Ly4=rsfY!3^H)?0EywoZKBjvx12Bc8fM6mp0?EqB5-79`wrNgIS+t{G9RzVXX z|A*R6mcrY!zpT=^7}F+QGm5GAvmgs`g2TU-d@uHIz5DSuGU5c#!+`kPzTD5dTnN2o z9k_56(C$oKX+Xci-dHnoN*%uJ@?A4NJjKLPt8KlQ#F#chL$+>5@aaj>+-X2(8l%Uo zG><|f&m`SUl!-rk@$=ngu*(P8Zs5V9w$9N$Ls zC+?(f7o-=JcQuJRZxs8liX(e`Dhhr?l>ApPkFD{M%S6epxkynbNYlK3 zHSC~{F3@3IE4$Nl`OTuEde+%7R)h!g?d_JMPX#TiD7b}qJZHhI1Ake7(NjSWL7@C5!hw zZ_Vx&D>A|{xfZVtD*y=GEY;>{tdk~4>oZWbkQ|#3`IW(1NAMM_Lk!b{oG%|~ULbCr z5$O{!u9J2IP?x>%S`hdw5lXM51{&Ta>}qIV34RLgN0`|B786qz8+s;egd>WK4uRvkW}Y zH9e7bRThmPVFgDt3&s=;_qnkgB&^ISdb)2IsTZz~RHo-2tH;AnpFg{`1oBJzm{e(GiC=b12Bb6&n5vV1~}%BpKj9gr+&?QzL?~Q`>PEY@QkN z0$c7HE)CSVA#)@l$Vhrx!K;lp!2kg3r6Ffg#-D)Jx$joan~^1S+%pK1qT&;WA-4|U zy3PG(fJ886*H&(Fz(*Cb-zyiqr~BRg!*BMU_KE@r|B6vk3f~zdKdOZy%aj(7D{1d( z#Y&Q!7mgw5zQv%_ELHyzOka?kR#dx~p zLlS3$3>|YrjOT1)>^(JC1DT-exo%2LWUJMSBrWJ&>(@<_I&xI^58Ka_#JpojL+jRm zZ^xx|F4;4W4#!7EsZCIu0XEVjj6!svnvb5=p#vW! zQog1I`3NPy7tq_R`&vUD;8q$!)FBN#0Ar z(HX3g$Po^kG{~(uE4&ok!+K474Rta+(Y+PX08!`PJE(Kfkte?-3?B&I+lTNz>yB>Vj;fdd8Ti>-NfHUWict7_%B#2)oev-C2Iuy+ z(7gdt;Fp2^O-QYtj`a6)#?p15^r-Kr)NHN?Qf zzx8%nw48{g`k|=bRXB9SADw04mh!^)Jz*+e{ZArY{k=?~LE}W4{CXnqo!82VE%r?Z zIlQx3U%JsYOCq&%B`m)V40f{hytke26_{+_G_QLG*ETsOv$*2)Sj=<7396{G_9?RX z;+;k)EH_P!A-0()T1+po3~S7fq#ex7(OlhPDmsj(Bp91w9FwFr~j zo$}xYgv&)jk%wizh-qXPr)ts)Qp+)!k0N$w0gf$iDh)L3S1kiEQxFH=Ey<2+@>N#K zGp3Gsj`yXBb=W)MWFV}qjNu}((K{Qat~Bwg8?HjR%3-T3z0Tp|>y2yYX+W_IpRUuO zx~s0~*c4$Jms~YM!YWPJo^^_qIy|I=9D1AtsM^K1{d_dYPM9)~>>zXVXci-}C zI(*h5Ty6%uAnt+=7~=toNol!pzmrggMboJ$tKWJ;mI1WJY2h(+kfGu&(38f3qe~6M z#I)X^#b?dp+Ohbt{|-}?(D3;51W8B@F&0@03xa4h>p>EtZ5#39(-ToP?|mQSC8Fvb zv|X@mLBrngd3R8MZL|iGT^Qv&TXe8oL~g7QcE+_we}ri;?D&XpZRsk?jo(w8K-1n@ z<$N%>0LgjIDep~V{ezt4C_jVrvR0$N*UpA-k+7RYy^5l=2i%M$*GW3Hwotvg4rI75 zN>J&rGZJX%#US^l= zRL~f%uv&)1q~T~a^1^j(6*OgUxIUVq8*8wgRpHXDZ2!{3$-^V5)E?tS2GL-Rg?!a+ zl#SjA{~b6=EdLwrO`*LujL!xL+jqk#L*mf()G)p!_}cuoZ+??Upz-~{UI~{`5}xF! z6D5#=uA}Q%yn}19WntZF35enFuLJ0e*|4<)tdeXy|l99+WY`8C@tFojjw>MDM=7T6-k8LaBEW!8N9?LER> zqm&a5Cn|ejgR5|s$Z0I9$fYx1I?CD|O_Mf=|D#OL{<70^hG`+;s~O6W%fy5)V{s1b z?>!CxQeEbn_W^}_&q%w<_PLdX^}`3RLmIDWWKLxttVF7n6?INi_TJ2JT3^k{GpYa- z@{iVoa+826SjFt1CBW>P^Ep8?978~%G@y~#9?&46Gn|R5uP|0!cwiDYjED(syMa;h60df)DCZINuN7MKgh1kjWvb^seD5a4a;bCorUBlARO3@^P& z@sAT=V&_5nPd;G3Knmpv1S+@%GlniQH0HvF4%ufYl;Q&;*Ni?mD;H3EwW`!__QC|h z25Ez_$I37fXk2UECo*kGxs0M7kQ1uGL4=zdn7@1Mw{GC}wwGUx8>oUSwV={bsMwqw zt+My!=oF{!Q@iBf>r)y3!Bn1EKYBO8ym>^3>i7?P`-l5Z{D-~$!PCJe{=1x8%cNRnOex`1)ynqE z(pzttVGgRTBDy>{>QV=+vFs%{?;|C-ag+o@K zcaO~o8fpx@u{T)PmGHTFTRqi+E2?P&ga*@AhZDrn*jw^+JC}Z)-dc_CzLCtW-VKMIpw~o!KBBbnRmqNOz zy>GRq@>$TjVO81Bwwc*O8$(uc_4(mohJA=XS1{yhf9&~;fy5COEGU6+re{`} zb8?B>%w`ps+B3=pCX=tc%fuKMFZe>HY1xrg1*-=u*WB!-M}SmZxS=V$g=Qp|H$@v$ zkw`-zylv@;t3VBQ0N3UVPp3>PbP-|%kPK}X7@kXHuWLQ zMQ6*V#Ma;#Yh!tQdhOZPrd_b9p_5U?;8nd2>5NZH8uO^`ayA=7Ugv`r^-CF7or0+G zKI0VxdDpqNn-h0gM>$?7SR%uAD_Bs+0cE6%ZgFmjIu@)}Qgd|W`QAq1K740PD<_CsaBTF+<&g$&Z>7PON!+uu3z?0{?4gNg=q zk9MSh&>*7~n4k~q8~|l@RW}laf@>R+A_{HCrij}rgLI8YyrFA6YQJb%?Ymm7GVf){ zFo@6XaV~G^u6pt&pyL_N&S(};JiRIFkyC6PJ#$-LPY-pV4sk-MB=~JH){xeOnZ-A6 zIk|^BtMT7U;N#?LQKzY$#AS(7)q=X7v6GpWIgb3tl4MVU@{qRWa7#(}M#zc9@E zA3qaX-Pjq?4{L#%IntVLSy`OZ1eJnpY2stRYfV06-MTR`0@Sq|6OrIg*c!R1@nmwM zC97ox1N37^v&hBu#R_eB0nXL`|(?s%@ zTR%jjNGY6ElqtX5T0)2Ar<|vmte$e7V{(Z(zsAvY2I+qXYM59XayD@`H)rH7mq^7T zsaqy@u~0t2QjxcCD`Q`YWWkEKx0-Ol3=Oyh)ibNF=O8Uj%qNGZEN|ZuxRb|m|&}l z>Bv5OSOMbFY>bKuc#73PIxm;FA30FhGn6uC99Nr(?0KzmlTg-)lStENASN>KEs33g zS0#W|N$6Uft|G@>YeK-a2*=?S&S;e5<1hrz9ZnVeAKS*$SL~q2 z26MT$-Zsy`R@iR2SKHv{WJlWE{u2W0a&=ET^Uu2cf5U@4C;#8x-m`;)js53+JeBsJ zI<0FDL}}6v|1n<%KL{Ii8I;Sc-2FF^p0iaHpKg(*pe;JlosQ|ND6A467Si z(^-BwUG!LmPbk~7vZ5-eZNxh#^EAeDoNx*UIp`+QSbIMSqbK+95m9g3#i28i(k+aF z@g+{eGrOV6j+jV&^oFZk5xE_=|EGkfH*A6yl&vwtw`|GOL2s%_kG?@Xed98UTD30n zMvEdh(6x|Im<4HcA_847h^xiN&|pQOGbbEJ%O;A^JnW`S2B(3<)fbY34Jv1hL_iZi zXJl=cJ7#tio|TM_3_?(=boPMT@oN(9gym_{)7XfmT*}@S+F{&ceMeHYEBWWxkJT{p_ zu4V(XU#)y()al4>JB949oNieP!!8upO{D0tUBL2e(&uI=8FDJi3Vf@lC8eyP){i*m zTuo)^0x};#RouEJ;qjcJ1u$HmT!ey9vY3x>W|6Q{VdY#zR~yrLaQvYaMn+qujXUG{ zp)}S`5xeeYHEmXt*|?V`maU{Z%1C3WfsXTLhFxaOf-P5hP@qqJYuP;OdRpKAQ=AE> zxyF)!n*0CZ(`N(c{(mq$7;f(W_wlTB|CiJZXK%%lXd*7~6mKgyFhKZ32>L3|W3rb) zD_&ZqJ1C=t)RcGE-?{x4Xn`=V(7an;;5}Cf@?S@Ji5S1Jc;vBrTk966)5jca>H&U&Fl0sJv8=}sf;y^=4g<=pcZd`ad(l-LZqeG^0G zZA@Zj-NQ50B4y{$Ul!rCu^mvI{g3DVcQD-OfA8g4 z>HgWQWsqutPOofg|!F0x!jT#hd$ww$8~YCLiCC^_blpXE_+zA4xFG=o); zbGAY)c>gX59OSHbEz$G9vk=TJA20CjdTm{9{BB&18I&k$yjn5*;T>2gwV?lb+RaFe z+ZzDY(f?hMC8VQIkGCYo)>#D21`9ePS@bWsR_l}|n{%b?ToD)Z4>C>1 zIYfjvl6BOlnEEcFzARG~BqI)(TncW!6*TM(4|@X$zF))yf^-LI^j$_4X?}$M`Gl{P zC%;&+y__;SE0QdyqYBF47XT`hu^9krgjsnArZ~!vfZqS}$s)(_Qr3icE<8N?ifd{8 z)~fC4|0sgo5iO2^TK<2q=f;0N9Bk zbLszZZ@8iV_wxK<=)YY-3zqC^cR}~>&?wm%$K#eBW);=%&_X%QqN^yzQyd;EqX#MI-}6Z&IB1%#aIQz3@Z@(tIM{;CV`KUPhUEk!FM)5ja$DxO86q49RtJ z9c1D0>4~ny)=ExsLjMjezKDZp&c4n_PJiI}FEz^S<;m#_`^D+WOZ_5SB**!?91Ahh zBXo*0$niyISVuKMl(3TAO*tck>H*48kfUoH$2|bRZeT~~$y)~YIJtQOf~wb1vUo2W zIG;tiSm`LpbBH&=8cRsXfMpmh@+g_2<1{_a<|IQ|07lsd9fAZL6 z5lPtB8II>DN^%;+F(AlF@H~V<*NQ@da|oA@S#w5?bjL$|yTnDk-X+g3jJG%ylJT+G zjtRM3q^NT>?4S47NV}T+J>66(m1q&T(iLXpw;6le70QD zm;1Eg|G0ho+T%YQ?s?)rJl*(z+{^O`)OnS1rgGi|R^OK{m*;kb@kH*&a=R9>9pA_t z;5eR$zil#n7bz+wDQN8e?o?PR1g3c2rk9QFEZ1yhSvM3p5A%O)CHHV%)-N0nlp<}Y+c_?f&Wkw02~||W z`OdH4e00a6q?DLb9Jf6YCNy)iTltJc?4pZ2bOqNN6aC-7oJR3N$kyDT;YH1NAIDlUNKQOUE6X_HVc zjW#3Y%OGwxzWkQSiVMSd)q2}x{-xo?Q|nIJ@g=Y10f$5sHGRaTPP|*0{&z3WI^=&90iD+yf0@ImGtF_LDBoWU;=&GD zUku`IqaegiR~d1)ttZhRHkk0b!U$U{u|b4iBfM~V)dn$!eery4fQ}6!x9>3R5bMi- zD2r=s*Q_&rt2VSl$ZKrZx{acVU1=A5eM+#~q~FL6M5m>Ob`FSA)4CE|l#!k6koLz~ zPRMI@5&!P*V!_Hz>{`>-&Z%fy*O~1ULJOxQAksp0$HvaZwP01}DhyD&Kz!>m60-t2 z=QN;rvWRm`9XwJJc8j=l>j_b5qb`}%22{Rjc1)&9xcN%{55H%VhO@jsCK>LANxqa9 zNS*!9K_ULf{_t=U|MgxT5C6yJ`el+ASU>q**%3S6_42pXx%s_1cQ7o&Bwt-+s{tYO z8yQJiuXww>4%cFjpem0lpp>r0sr>^<@RoHC6QLymMs<4<6;qiI4i9?8=*1Ia5Xx|=CrRU@Qz z1{tu&z%nW-nP{Iqk))z!Hbss=dDYNS`W&u5|72N>MmtL;KiE1Wrt0&}u#{8B9Ixe7 z;MdhQ2HRCo7}+V-V^?LeLv2i|tTo%hjH+X7q)r24s0Lh6v$>H9-YTP|nySr|`DGY4 zF;%Lm>u;}jX=lI))}Cd|KE50zn%`BZsb4r@_6`vHrK~# zbOtm1IkNXb8u4X%Ck(zY$av7qy*mOHdvDM0!(;N>v+7knDl+U2C%HDR$XRNY!`#R; zz7>ROUb7NZBaX8b1aA<68Kr4%*$nD8%{a?_XH;oK8CE~IIsqGJ)o@Q2*Gn-< zAf*jHyNuGm<1CuoGy+wu#zA^zG?im$OKKX*2)f2WtVJ+Iok3;GB?<(s7x>id*_}Ua z`2R6UXh!0g6&FT<>MX6g0@U#TdxQN$m;XQ9+sOa#ZSQ|K<#$cs^qC37pzWuN6$(}^^W{fG}vuhgQ+vS;$Bs{L(M zDz14OkG(Zt@)O`hoJ62=KKcd?2YZKmtstBg>OV8e&aCof~rPa4`Q&^E-ThIAltp}_*|DWy;_MP+p+2ON|{_kF%yE*?qW81*;9r?$# z6|5eUf^xBTSv&wGW2uz77Wq}}3Y#?2(#o(A-VJRI+rhd|3&g6k@*!Fzw&tamnbpRM z@UN+WHH-=QHHOIf#hV5&|SVO*0$j;XeTTqarogEQ~P@-AXpWlUQ@va6V ziQ*-OqNr{x42xft$%^=;)yvY+q6)P_3%e9N&$2XrRvJsRYP+p1?vg;)ZkfBWpJBruhakg@L=DS{~jDZ-Q@qjm*+0&|0fdO{?R?Vok1vwXFns??h8)*-y6?h=BeI?!TD-Yg+P zY6EQ}JNletN98l?Ss?|XQJaJ`7!F+h&$Hns|IfWVtI+??-O68V=0CWm$QXr!s>nCn4_Q~_ z!*?CZA`jeR7h9cHOQ?$~tKYl6NS%ZaMq$*F6>2y^jYYprD&M0b$=m&=nxv}Ab(ZtB zOO_{m?SS!m8BEkscX6)OIWVyHh=#> zRDkZRtXl`j;V+{Ev}$g&fDi3yPygF10BiMs&kFV*hliW^ulMpSN&i0s4dC6J0gevv z?hb(!lzs?`G)kFS=BK)bzqO<+m+mfl->S$wwYz@_Pb3k6^e{XclN z|J0%Xhr?$Z`;U8hR-^x)y9Q8h;6Jzu&>V(>4$wQ{4_OK5MR*-rKp)^GoW{KPwa^1r z*S~i~pgsg2swS|OA8I;Djdg*IYTu(W&_4oAwSkqjE7gIuPS5WHI!dshGrw#wl_Gh?=o2LCyPo4bd>C-~|hiAi$|IfWV zCH()HoN6y?Z+?BydH}Uz*Tg8B@LXYl{`Nq00=`*Vw_ZTEQKK99Afx=N8Wq_AhK*_# z^8!5gY9ks1uhiYB42@rjdmWCI!zLpC-@j^PIr`sD2T()*_YMx-_-_aMha36Ny*w+@ z|Ia`VaJOfGqYt>-Ltq7+Ktn>kESX-8NPpfWdOg`$K7GH;>?E%Zb6Fz#JPWFNf`RZrB2M{YA|jrB#1A>5Wyk4)VY#1$vaZdKbIt5DaGzHO1e_td2h0nisg1cNmeRIR4eDbR&Z?9-?ECp(k7Qt z2DSjUQ3!rc3c=EeT|p_>2xeQwV5|Ed4JejDH2YUU9EAaGAOx-*i+GW$s1Myn-G*L7m6m7HX*h!8$Q3A`LQx3pht7Z9pE2`*A&yQC)%c2lV7?U81 z`RUlM`gM?_4A03G4twYncu%y*F)A){Pvt1dDGtJ3r+jmLPHrqM_J54&Qt2OR_W$0& z;n21JJviLx|L^A!dvQBRLbT1cwz(@~OeF!>Q6FYO7UBele{~D+8+?X;Uql%WQRhXF zFn~Fxb#=c14)uD5ld|T&0VckKI=*cu@h_>{LKP{VHk^=Vcm0iT9;hYn*nbut7%ZT-Wb*cq_J3~LEWVg;7jQ|nq#t{TIylE zY=Vjxii6Kx2u>=c5^Ix_%}7ErdFQMj45PU0JG^4Ulha_HHix3Cpw==A56lW_MYr?@ z+6a!`_D?sGiia<$@XYB7GKv~#Ewe&+ppB2it0Pzv(IN*RbN5q3X zA|4W<{XCDWcKfL`jZ?eaFs)ixA2n!_T&O+9ZE$UccAAk9UV@W#NEipRAz&K<_6H_l zd=uOq`6`3II`QI=*GsyJFxN`BGR%3|Wl5K7;ZxDXX+UQk`~I*Ya2o>m$0KlZL);yS zD{ou=xf^O8*##e5GQzc`$l%Kp$VUcCTwotYrTONFXQru5ln%H z)@knZ)ooyxmKPxu+}nD#+=d(AX|plphW%l$wJXV|w&raAG_?OsNQk@gXA9GRc7| z$oCgZM09rlI? z{jYkzb1>NCf4i4w$@5=RMZFvfLIN)fpvu=D z-fMA%6-XEhx8Rd1PPRs!%yEzj!6Te}bw{%4&*}?)P}LSlO#&K`L`ZCn7e#-|mH-{^ z%u+Ag>a$qGwhrUwHTWs)9HRGqV!{Z>g{Dyi0(XQyb_G&;tq#kkF~l*Z*jeSQcHu** zz`Y!lKH-b5zU}6-fCtsb}U_KUhOu1WmURJfUKgc^RH$(xk zS2{WgGGZT}DjWn0#-qBX?(0PqdRbjx+l#0?q*Z0T%wTL6jyA7x#SczwJ^^Y*X?oP} zx2KB6gtd3dKQek?01_G^87&X6%A)ychT)#k?=)C08Cg&)YE*F4<1xBB4>tSd%6cIup=#%Bg7A&J$8YO&cBl zoSb$=<4yx6ZA&dI>L``b&2eV12LitFy-QM@1r&!z&j-&39%A))NorL?oIq(+N@SYQ zhI+Zm{cXfmk!Z9NS8k-iR;)E)s^qI#Z$g-k0?>lK;8-7;~1pB zK06q=_CHVepKkR3_wul%-~VdsJDhNC2u(6FM^C&n{Dgl5r{1G}KMSsV(}>O%qeYIh zG1sF(XuATJMY6s_nES!kqu}|oV0btfKRX=ZgT29!{S^)eqi1`A@!=sJe!Vvt2K~9T zMd)i$o>-1A@gzzje#W!axAl@hc+&9<*x)GXf~xNSLMI6#S%@=4iTp@sQH~~2jL{^@ za*8q>glIzIm|RE66va_;d9)?F%Dd_IZA|9#MH11Q{tUH1 zJ(BvfV0^jtRUZ<|e303b&LIkr5Hq2pZ?vH0MUV!gD2`}^^WI+z={RpaiU?HtdqaQ1@C(;&mB^XG3Ef#?mogN~<1#q$60y!X8Ky!+f= zx~bOL>6iD6c01-f_l-me6B#qFy_9+d73x|Pza8)9?bK_4E%9IUzBP(#{bY8lPrS z2&NwyiE|Ic$ODZX2^Ag9@Fc_etaHRg+vXQ?&Fh(+v_}HUjY%8_Y3}J^p-5O!CJhoC z7aJ^ZSk(;TD9B%sm}GpeN=x1rn^izYLFO6JLNT^|UgKm+*@z7XH6?K~kn$qJCgHL1 zv#Ju#$61uJ1N@x%_1bqdkRL2%D7N4pG5=RL@pxL5o zo3Ucoq7zkX+e0ti=a-7=YrUG76ZMPI>W1P@n+F;^35PO(yyahw?u&!hi z@{{3|y<>2sZU3$t+qRudY}+;_*2J0Ewr$&-XyT+}+nm@vu~(kweb-+9^onxMrB+B!5F=_P{Y)Nrzi@BfUc_ z4b)-##rl+-efFWfa~dmFN`46I=YGn3>obUgn^%l7Qp3qMg|g$IgwXkHUw3`g%}(KmB%^_pJ4=X2T^%I%wn^4l7SQ(6 zu?11!zxyMq#eSLKlAVX`3u=gYqj!zikUC@~QBYLk(^P zTYhcPIr7mS92lO|K+m(*&L&+7#YeW{A$;(o%?z5dTc|VJ;cN~tahsaNE`ZBnFhMbj zs?q)Xkh#YaSLZmIkudZA(b3!CZ-13fNy7O!GoO_GTCRV<_S>Ho@47R@K)T5fbfe}z z_bxx4i4o@l-c#Vs{20Ectf#D@H>;m4>6>=+SQ zt28GoT=lM`V1{pDz~qy5`k^gE#q21N|CUJa+*%?hXm-vuG&3#!7pvBe-1%KK@=< z{Q%hg}eZFWY3UJP*nY$_$5{ zS%CQvC#x{zJCa!dzUPt`Wk5FY?ax}S4qbjrcD|h_RB3X&r3X<&fu5}~K?d2qD>e;QgtOcOF)k!Pk#ieTWYnKV?l74hNEVA5L)7S7hTMeA^zvo!Mm{Mos zkq$0R`o@Jv53O7(lxNxMD4U4^{5aZ_r&eu}c(pNZ=Y_Q?J@%lHt7aoVV++*l*$GqI zN)EnerdL2^eUEPG(=?0-#5(|!(_1`|m>^sH!ps;Me`>)c4-`csYsLp7V&8 zv+RDD7!MO55-K+l0uRmy+i``Yuc8r-2CP+sV~rh7FOYBri9S^`G3lKwb3W;2K!V<0 zdHq-!?6|%xe7v0f^zz?jxJ(tkQtm$xXph68_r%>)15^m2yJ;GDg$_kBIWGjIDZ zo`2%y{tPlR+j+T|=NuJilUZtzSt^KM6_WRY5}R~|KE;(CK=$}U*jG8MJ$Z{OX<6@Y zMu8x%Kr^hxEopX#Pqc6XcMYOI=@Q1qKEfV7%mY`<9~+EMK$diaF7-_aaIzC(MkCuU zi`kS?=#p4UF~UP8T_bSft zy9JI$$TpMF=tX-b=!YGwQ@bS=o`v z*pYw^)@MKQeear9XuYN3=Z1L*cJeNvYp$ovwHy9A>S||uvo-cUxA&d4Ve_tXtA#=} z&359}%pHxddy;eO*U!!c_k}r83W~y`T8bYdu6I5U?l1=8swC29CUSrGChZB&0w4l| zgKzn`l<8SFN8y^V#K+sF37Z;x2ScKGc?o!I@vIgLjXG{=$fY9IPsAml zdLHL6cF@?#0FtW@*AY@N@*L>5d2M*6`J^U%{bn!7R-#IH325$=>SN}JRNNU1S)*8^ z3~#i1gbKA$^vxByC|Ze}qw+FhpuO?zAz`)K#@AsUei{VP3Itq}XO;=-RR! zW>o62%B{9ebMs;?CKj=w`swS*=RfchO22l+ub`2pw_~=~3W<;Uz$YQ&j=z|&)1ID@ zTFO8dgHNTO7-?yihrMpcs z8|!hiO#}S6Ut%ix_S8R=I&-LwNHkIYm(XnmeLs_%>!9NYt?$0|TjA|*td3+sA8;Rx z>e!|i&7SCPd^XvGssvwC>nwhqjQVtCXr8&a#Ijss@=G6-Gm>@BQY5sievZei2CcP6 ziTd-Z5o!D!^&Xl|Oh!s)p7Ac1$-bs)Z5MD{ds;@~;j9_Ro}tB<3ztXgDxjlW*Iacx z9OqR0ILrK0pC!QiUd0pl^I4_R^=NvbOumML&$f~yNiH&B#_5cAd=i3}UnQ3O&Yd0S zmxcyoMiw=7O)qBmrrUJk&!7eQ0cNus|H5T^jE{SicL zf(f>xh!+|oY#ioOeAsAGpo9wK9}z3=%RmXMh?i%z7SK@GkCr6KUb^wJb>?NDx*OVOW z@&g5ZEUQ7^nKJz~bp&)$O20a*Df6Clmu#6o9kWW)TkE%3w}9O!Xe}R^VvgVYjfub! z|Ie-P^FI#T2EpipRK8(~9=K`SP+}O|d;PFXzx!a36`!P>1|~T59S70{jHGgI3S}V- zJp2T*us_O`5%pb$g24s-1Hp+1Zsc-f5=47dyN=P2C0tDs_ilpGR0B_oVfRL;PvS(} zW*Yr|A9_F$obPHIw)ZvmgqU{6;4E$MJ`18gG%nb+@%@_4tDs)LGkh(tSMtVZ(ignm zm$_y1+v@I8ua;G*Z;b9f(&sZ-C)xI>a))Y%4xivC~&U+a+*boJyb8BW!+n2LXLArOhPJ+cE zB;9L0cVd3(9Yhc>9*`+E8W2Hj7*N6JI;pT3J|Y*x{`Hee6m|p~FVF}-c@#oo>FA0X za>OUv1$(b?b|zfhD$2z?@rD<+@Fg6DlW&a+v3|$1%?#w&vup*&PhfWh=-^&y30H_- zadQ@49-n0Pz8=CZ3=E_@0<=t|LfMGE3!WYk6VAx7Pot*?+;`Em_nBcf1dcEZL3J+LAj{zrKyOdZpdJ4CEpu)RT})8u;5MCIS5HugsW2Z_ z*5k7tc_n}6)93y9{rr5wLCCQC{p0dpZ}ci&Cg0fY(`dp@IbjQV?Y#)T>6NsTxarlr zBw@>tx-*|%n$LD)gB9MO`|iWvxfbK^D`_wB-&e>Go4cA7fd6auCDY{;CkTOKQ*g5H z9vC%W$*Wu^%<0GZ30cXjkms)vy!rP-6#H~lERvmGV_QiG3er8|AU43tGK8hDlb4Ig zz$X@jRe*%K0PN>fEWLNyNBx)=_%J4O9RM3g%Vd9xqs|t!Wa|YXjKh4Ny8R&$9^?W! zC(I=rlawO`1O*kW083vP3`7=0J?VO}0jy@b4h^KxWIzriW%nYt9=bs07h%8x%&v&-62QjQ&oE#}5(s6=s~*}e zk1lQNJTHmbQ9thu&T^oxHEss96Lto4>k2=~*U^fWli+?vgkBkmgM1^M&*UVaxF&-) zBf>6MM`)Z_KETy@Z56xxDB9;)H6OhQIC0 zjG&LkClL=^gYXH1^9B6bDj*h%6RSrAe-Fu2uLNNerp3(woiBIjL)g8r$hd2L88zna z=lbwbwnSE@E>INC4d9(1f~a7V%!T^7h5bx;}{xwvM46f9aAJ#{YK{ zbRG{#g+N0xeH_AV&U;%&HjNj*Mby_pDKcA~@(R^XTjmqysFtPC$mZe9g7vIyhYU3$ z4I*yJtqWsGBx!wJ47zA)mP9UGdAPzRT)AT^Y5Pv(Xq|`RSalM#`-7GRLP|@;q2Gg3 zq7vSnl$)20*V6e z3w$q>8Ys1FE|g#x=_HQ3!UaO61z3ayYz;kt{Glvqw0?|hh7*3|Clz*|yC0p+jb;kS zwh6svE}MglJF*PModID3vw)a{$!I(fnL%fBvUCC?aOIMIA2Ek*q4$mll`0Y#PE5b{ zmkcD1!U;&?5e-Nxf-g#?k0jOYNP4M+#t=Sm22DH$Ap;c$i-17U`CD)dOGZ8IxbLCZ zC%OzJReYy2LrCJ6%nC;W;3p*!pM!*HO!C;sjs52n00%_J&HscVLlhT1y`*S!3*Hwh z&q)K2{3y}2^eX%UpL4kMhX-*lf)SRf`C?S9K$Hpm zEKV8pXF;5O#00cW zmCGZDXU`ff)66hCD!$cR@3Qgp-rr+s{mBQh#`~}?*);s#ikT5J?B(Na!b@)38W}4^ z#WHJKCdf}>@&Yksv2}-&mc*r|#|0kT#Yyk@ob$TBlI9C{tvl3=c2(oCcvdsUClI4v zJlLlXA>WLFo9a2iI;jU{RF%oK95#`WzbC_fC`9$ns1F%=XUk4tTzw`r0?&ZgiB{A{ zy`q8*(9Fy=3^Z@zYLg#t0pfXS6hkKUZBu_Ir7R2fbYf%3ab?Ka%Z0D$O-A~w$r7KczK7vI!y(6&GsRM)3<{Mpt>0rRU~UVd?qIFor1sp@ zQT6y1MO8|;ft1pBRpWZ5?zio0t41R<)7!TYMR&Thdna>iwOf z^5N3Hq~__Nt*nB;IvhBrim`7vaz|rXmHj8xzYI?3tYGE3S1hr8Y#wS8xugQTDBrXu zv43}wn=sz>&)s)E{MZ|hO*YAOnvDZaNOHPf9zG9`A44xQ(OaGTw&!a5Ees=NlfVjr zQvgcDQ?ADepL1D;ujF(}|QGF%6CTUn_q;H8eOrcj~;{oV*@H&vWzc zmwMdhuX~j{yBE|hAytV@LBR6AEV$o)JEMr($hAhha_f6)-4fKJ4_qz3g(C?P@Acy@zf}1x z6DJyIEe-S{C=lklYIN^VuY0=J?8SxJ@lpxOtu>@J?P21oWjm3MbzJX`i?j|V2o61j zkXW~UM?XJcI^o0T$rIrRvmKz?7Bre>O%t|IkCr@44T=1%oU_F0 zRd8pZZ=mhGmXEY*xubo^*fAy|C4^V&MBkzD@r>pR;8m;HA@Eresnwz2!BT1y$F5gq zxKMsY>+}#;7^eYl6pBYjuJ6pI2qvb$E6Ms8(fDB#lH1~+%mT4WY=SPL9!KKJlj--f zrr*E+C4!wfU-UXWcEvjVbe|>q1zj7M@muw-SK;T$*F4V6E3hqVXs$(0#@SNS_|pSI zH)MtZ$BIlt0uqC@u-7|oxH=yhzEb2JbdKY1`tj?wF)2)M%h)|ZNTx#qtSsLqJ-i7r zsI*$=CWh6MclBu8&i-_mE<5oYP@k#RpA?MEn`xW4-X75uP`ys48I|*k+PBPd z%~cd?RwDq!ourWh$~9D|;Fbupp|yYN-I|3UFPSbb{W4v=e2h=jjG{(Ye&cO+`+)A| zj2w~u7Vq3_Jn0QqZH#!Ij*|l0tDm_V0$cNxi2RrLzSiOb&L@JAFxxYZ7G^MFARkT% zJtWUer@Ul(_z2G1%(N>?NOFf}uZ!;x%vG+4B}EC1eA2gS)siU?mIhBOaEX*p6<)|b zeL|`$h;?r#(F&ovzZF%2sdnSs`TLN3YGYDSWuz&&oM1gGMH4GVQ{pKTtS?5Rtm~+0 zw>fhHLvRybRO*CqbuR+Mh@NJs#4>Vt4N`E8MosDjpsnylS>fR`;&`-@PK>5njM^ql zUUS595g}2YA@}r$%xGOi{ltzA`E*keK}|$%mF$!f>?oCY8qTQT5y_(n!iiOr>kNk?m| zhKZ*`PT{1Y^Fj-eFpbjC6Y`js^ z#T0Le#(ek8V;E1$-W(i9qPMt)k+eh`;Svoon?Ff~EZ|7)CHAghAgj^!-sH@BXkLR_ zR|9IALn&qbdQUq=B_I|~NPw1;?!A$e8?BNTBo2fCl>2MM7D1r^xYXoetAh&E3Cyet z5AW}|ju22`uz(bU87x!~;c>K6&A9(_~HDX zs$IOL`4Z3^%FrFMT=u8Jcz>>ypr_-X0W^{ozuQ%yJ7l};Uxx7lU6U1kx(gM37D~`} zY%9=rI8Om>=U3uYuFGdbIbM*lPPp{(Fv)R**_8d@Uo}zXJhhfz4mt?N1$5l$+;2!K9P=EJ)@Y+b)`URhpFU4Hy> zic93qnkxOrk;hs|w^Fa}kT>iTc_iD%vTOc>nlbVIO#J@y7|YwaNsB35ELT0{G5^;6 z8}Z%(bNl35x1$zL)dPZBkB3=%2s>Rab^J>l?D@*9VTAXlhli;tYFnpq6(4)C>L1{d zCB}}TQDhn_6Gx${>BWK~4Z~cig}U{+k8WW$g9=xWAaf~4xwL7#P4$AzlOa++N-6}hLN>2k{xT#*IZ(3X ztpzzyQYdL%*UDsESMG)<&3la9OH+RV+Q@HwFq*Ao>HILQF)*Z_F&?ty)i4QYG8wd` zd-5&Xq$C}5$Ylc67+M=n>$b5Y%he1ChBi0`WiP>-g2kC~f?*>@ROk_BQp`g7NY@>2rs3c|6>y5pt~RSKNO%~(!mw&Dm+u|-CHuWjkeB7WVF22F_O4XLUU zqgYucKw)6YPT*W^Ew=rdJ0z;)!pV$ad21SopZ(JJLs_zENitZrNTU|&*oXt98dMvx z&l&|icu83FWwQoKiBKZh4Sy1<b7(YmZB=&&FLfdSfxCqqq$V-a$qKUo(E>*{Lx z)CoQnHRLBPzJR{{aXcNyD!inQYN4*fAGKK;{BYJ2pvGMv@o+HuIbvfc}HiXAWvm+gn`}VM;NOL zxn*v`*zh$X0ZCZ`k!=H2T?tv*1uXy(1`PlMF^>irSMyEkgps1Hffs%)A4O~xD;Xi1huv~>Qt z#{tv!jiC$47!pSTwvCeJ&1%23n!a|t&o=FfcA(YVh|3PLabts^Q@=kz594?D(&68E zKd}q9tM!%DOY^wsz=TWIrc%*XO3uKP)}MWDv8eS;m|tICh;b(hSJyPNEzUHd+%8m! zfn-P?MCIeSMwoR8|39AsHO zoC)*0y0Xx{BPYju#17)9z{<%Q$zwm<=sPX)3HSlhsJVJqgsEfyn#fAnXa@>?_z@alvCbKMnFMUx7QW4%;2$ zxqB=k?SP?!yseg4r)RNv>iZ)Go4Buw?=#W6IGXf*JTJc9ZYIBUEXzvoiAzHrday}n zEuUt1h{y1Kh0@#fs0&pt&aW1-nq^kP9S6^`SD$>|)}4XYIaaWh5?+gv(0lxi4z@@Z zQmn?zoy2_FUj)k_Bd_P|`F#yk*cmE)F-_8UG!_^#O8qfOrBG~e87f%#Q&TjLqfWNt z1jxQm_s|DZu3P4|wCLky{SkVq-)934`Z2bQIEB2LldEa_|_aBM&Jy zRb*?ZGh?A_sd+>jaw8ZUasw_)l6=a;S}HHi_O9%gkUY;l?sMvk~-u zSU=?Mfy0^5Y)r3|0|t6gX5xpb@M{X6E(BlDz6$u7B`d#9ir}~cGg+Pj4p0y4UEEIM zNm7zw1rW2I3>jIUv1ln;sp$n({47Ufc__uX7r=u_4IM@ywgub%zF$()09zilEMc7D zZ00N%0hk;arDi04Nwc-=itS~jK(9|s5@hEX{D~f2QU>4R%Emlu(|m?7YzYd(`9IT)62TX)JvC`hp9bU#uaY?B3+@#|~SlBkB(hJlNL|6ZZ zN#w^?gGOPK*q#~>^g(n++;`&StMqQx)WcXwN?VQh6AWqbAFVNrH)F1g@E=3);$V`& zhC2sj>?l>;F9VfretYUKZV9sR;_-~Bc(tF8mDq)81fvODNzj-t z22Iwf^pnglB&ir<^EGL-YeO9b`JEcY$$O8ij5P<3A!4k=zkmX?6Xfw^tHeYogvD;Z zh&IrX;~Qf)1UVw&x?2O^Pe=Rn_S!mJ;x7MOMxMqrtWj|iWdekoGhc23B2LlSvt0*; zE^m&~E^&VzbB&~S^$w06xZ|`V_wp5M5vMwa6f+&=Az#7UhLfyO)Vz1Hy^vlISr@Ne z6g$N*4DeUIieWdOR>#Xyf4EDH41c>Pye?+tlLV%x@DMMPU67+1P~o)7hDf0|)Vxh^ zWo8#x=$Jwt5~wq)#gJ4Zn3^8(E66h-Q!;+GVpWf^%ysCL@wc!GO^P| zV$PzaRQ(|1P^ZH2*sa0V` ztDo5Rod%{iKwT8=Wl+cCz9(dvPu$k z3w1>>BlbZt+D!(zIlrL5scFjw@u_LhjO0tC@!hcRy$`c85Ej=gd$UXuFp9|i?{WhE z0DODV#9*r+r2&1MaN$D$Bd?Xk}++}=lJ$)#O(K;W(msU)+?zPZtGnif59v6m_VfCI@%6(ktF(Q&A)h%P4{ z@MQdRMbZCbF`+g>#8{WFEirJ6-ZT|DRK;Q?mE;x(p;m2Rj zrRU^pUX1FtbXvCF=u(ZhLO5&UDGTV;QlK8wY^|H4nB5;Recp!*#ckuqJ~D)GnxlP? z!e4@LBlM~Iw&i!mOvOiCrROTyv59{}cU}gxSP8NJ%Kfv{0F8Wdh>9FR&!7nDD>(^o zOPHH?BUE`$iW7fIUGgjCWNEdPa9AMjW2k=+++@0bmGykLR$lH7S7n2QQ>^=>lHkV^ zwvT@-^`3dG)L)A^^Z~cn^&6lmpt;fAu$PotEUK(vP+6J)ZNgk3o0zb0<|Mw)*lhb= zym0puP>27>xGpkL#+E0`p53yKO>EYf*{;IAXx#ipK~Y1yn&ST`miDn3^}@Rn6DDTe z37Xt)94;3BhN@hT!;(}wb`uJ4))i}{fHMCH;sGi^kDkpO_{s;j*P%$B%(|P9jlPe` z1Tzw2vI4J!o+4&*GBNE}@^%x0_9}dJ7sHx$I8!j&C`ef-dIiBjHs&(o(#56So9Urw zv~m+A@eQq7_1pg057NW>t$yL_$RboHv+b|-icV_F5$SWJI29!UN$Xc1WlWUHx22Q& z8(2mz-wCp(G&y>iV#f;NwGtlo8qpXms6K@icxn!lA-?!yh4oxeW^jogT-Sn-M>d0-< zDF*1}I?@{Y08ZlyS%qXX-ROTw#dLZB`#_;Jn|qK>c0q~(>?yw0RCmX*=NFeyT)+Gh zDrs{-QY5x<_4%lVdSi8a+|01L#raf{@8v2(r1W{Up|qYy$+R<%M)1hrN}uqpdq4{n z)p{R;^UdK2`t=5Kt}f_xV=R}?6Vwxp%zZ+#2bV=kp9<;g@FKgXAgv+O-jQz+A-Mvd zkmUL^$!1r}?f4X*xcL7I_3g0sKFtU;*J?a@VbwEXNW+6vd_x9w=8_wx2xF~rD8<9p z4EABBOGse$luF#_uPofDWG=~A@8K{wJ=u+6FI_(n@XC63_x#F@3V{!Su;N2paONcO zccSI`iC;!;fa-HM@@&v}HJe?RW=ti1N6^7@lF&Flr?DsIEZeMR#4v%P zrnpJoEk!`ec|-w9_y-J<_+Wdgt_PkvfA7P|7vBQlBwChlvr*47O-)W0Np7{J661oX|`0|R|nz- z=@+O^N(I#!6q2%mSt2-3*h-HBFjr%y4Ui%Adr{29s1rSc^cKxSxqvM?PrQWvU_VGC z1oUp`ZX-cq)W8}kfY^X}q1gl>papV(*1*1j*@(&`EJMJaerThYEm}?k2A6N*K{jHt z4*r9Pj4UEz5k*Qw5qyjITKAC2QJU_L!cENty-58UL@vS*_q``0YSS^HMU6re9*}BK zP8Yb0LsupGx`tQ~ne&W-LGZFN)^drKv_i!{*;Zs1avyQTMa(3HM8@Kx#kr2`KK{ORi_|fGjfL7$%3p?i)>>|B zv$0lk^RQOJQ@qSR=4k0A%v(1YtnBtNP}|tqtGwjOS+72}3Vyd_COf%*y`EJpeH{64}<@=oNg!a4Z;a<74`<|3Ql| zfTFKp11-cQ#y%2T@b$e{@cV>Gl4z4DhR0Ez1ZgOtVFWuIKRQ_KS@7ngB8;tR0Dd@L zVSj+aTs+GJ`2G|+*gmE{iU$Sd_SIs!rBH^#&#M7_7@qMM#XKC*xV{S#4-hWCQRRWU zEJBh?*!jG^6A&w<=kqlk!X8g}$?zsiA@|}zE(lvBBWIB8!Mc942A+Va!lzmz=Y(Ev zJ6O(e)g-AF9)yS4BalKQlrs=bXlZAbqk}~L)DZ*# z5rJz6!eVL?kbySe9hkA{n!78%g{Z1@FR4Umr(A??M7?gh>iPU(iSbk@4t9`94sk&H zdzU|6cK_0ic`yL%$VW0mZ5h8WCxGV|jIeFW;bV+aN_J4aKnJce>x z0zl4ROptMYQV`iW$bhCV0;iw(4}oi(_#$wo>HnL+Jz?)Z&$U^1U&LPso@E4fXHQOG zgi%L3iNh}8%uvNVeT()hY9;d_Cr`66k-QuWy3mwAV=MGQtItOWOI}67c?P5}dH-8( zgnuVH!G0%kFaao+p%&1uu`h5`e>V=BgU-YxSu;+N^$y%`C#{#|j6;Kz{9?e{z66=n z#JS4o@TsLvz(Nl*QPIM}6ag~h=}G8sC~(6qW)F>4A80e09S37T>EcLR>t8Y!g6hDL zimA?{HfkawQ~WkD`8v~jI|`dk2feTU5QDOgzJP1otYFK}8X0K8cE1OTxYb0U6MF;fH@g{a}>xG)f3SC~RqvcSsrlrF6sc{J&A=K5ldAj8Y&6RT|6XW1H zrAK+zis@m8Z7nl@B=^C{0h=7Q0+}s-tifv4%N)=vChT_z%|9IhvT@4t7~3|pV!F1< z>99KtZ~2D+CCxJsYtaG$tJ(8`v!>a#ruhrQ!SPSfX{FpMi`3#tEyMl?!>Q5#9}K5Q z0RIn$L$pT;mMdkNr^Wa$hU3LZ`9By=6Y4)0F7)uf7!Ka${}aQ(a;D%(K#r1_5@aw8 zaew?1p2PZljNf1s3y`wp@G+vZZL)Th>YuJ!sFe673JvexUmOz)@YrQ6K(FKp?pyOD6MUABvEWi3A~ z;2*aGI4zUo>R^Wm1^&s1z=3WAU)<@N^tm0i;qRYC3z^ECh>~_&jZbfF73jeJkfvaI z=rts<7mN+`po%Rgg_`8vf@|9Y4=5mta}x1d%F7RVg`nZZTim}eoU^Me3deEl=>JK* zH`6?R_NQ^?keud^xwI^`X@WyUw|tZ2C8oyK?)&(KXiMYlf41wUG0wT54PBM#vBdrf zzE@3Awd^PRm{Fm$)`lk7@f9+-lzx0E87wNV_*wJRNgjJLk%LUAo-Q#Bb_HT1dI{UG zjNi72`SKL&B5ZgcoyYt_kbF(Hz8ulU?-!^gZ|@rI%p2Bn8W4uRq(6IuAHI?=B49C9 zZ(;*Of_lf5M$=>`i~!9)J7`_1dF_j`y%oW`R1+s04gH?lZT*#czYc-;zf-R-W|Q>4sTck$^{R?eY0Vy^6s+>{3heo-o{KRPEhtG_ z(audY*zv{avA;(LheHE$3>(dtS=7vQi)JF$p&D_|B$N)N2YPq;e(=vT+3+WEMdzWk z+AM~}%eq^1NMdLj9*p?eoKW@LR73;6k$B%C;vErO5RB9_=bP^6(SJzwlBWM)f0&Z{z(*= z(fzAf#|F<J`~t z)ujl_(giFjN(^Pk+!(j52uZ#h+zGV^R78cw>*eMUlGpNaOA2tpbFw zI%sSJHy1m_|C;RYRlhT!jGv?xh?mk-kGE;En6zU<8-lHgQC~CSn&y7>2?PXxdN8>9|dyM zi2^=gLb-0JBY%~&!0eR&-NRQOlGwgXRqRQLvic7nVK6E?A z?gDePksWG8W0~{RNc+UR^)`uor-NL5Qa`m1B69$pEX776aHv9MWn+U!&t9-xL+F<{ zR_o6!^V86Kbrrc<_GP5o6||Xy2If21-F5i(fZ*hDY;&%-(vascq$4EP?Lwt4y?w;; zRpKlNd_eSjf^+&?HNB}6UdwI$!;8_lJZ?q=Y5P1CocIx1Z*WXtd->#TzKS|y)EjjVb|@-#JeciIt& zgickl-XvFN%X!6l*?07rrDViy%T-grD4|L|?D=lV-uB~JG1}^C=g!*gp}&{Ej_cPZ zLbp2jRHUV&#X`hEV_GixM!SsIcGBPFd{9d4FKa!GxqPp;D;Ze;k87YVUW&@d#~XTN z>quLE8MjX!@R=`r<8W$5XyXY>jcpT0D7@myd*o5b5Y;xM>GF~~7bO^nZ^~oNqh{2< zW6TQR@31`LUv2Y8)w6T$q5_Y#z|TJAFr2{-s%sTWMv*KNF_kk9%P zpUqcSScBYpws3|&s5XB50;WY780t#0{+G#zT^;ZmBzS|pq8hOr8<}PFcbnYgF9*xY zBgE?gOJ?;jJaKG?hhbh)M;6Ex+|jyK-@C5mUF)KAxYcP~+)OvvYS`n!uh^35T#v7dyurRe)ol>W8MBpP}=K2^ye`qDl`vN$en28FC{ zpo|GM!7IPf%6es)CTMS~TJ^6XGEVMf+%lMs2W zz`P2iz8jU38r(}?bG9yk(=-2sF$LYQqiwDB&)BFFiKY&;+2D|N5`YifQImd#01?A7 zLI42=39DFw0}%_ee8e+SpFD%Q8A^GX%=B2r`#?3yR!JtPy6oS@DP<`k2*g?J#oab` z=6<-g^Zfy#BC~lX-cs@FfeZ&V)U{|%fLH$4%a8rLK@)i`^{?7_j^cEDT9_TK}9%*X?KgdzjIfazHl34S793bF+vH^}+Y~(*)U!^&13&k*jPg)kyp*GU6y)0w$OlC0V`7pPy1q)W=_XLc^EthkHha zND8gg`8PuGI`7+kW6D7l?}ipth}qs|asmu=;iw;2cIFDFG%;;sGX62Q!dEPkV-Z@kSq*i|dLu z!WsJ>AU>7c`65f) z${iM%_Sk>mY~2bn+eHn~ix(?%F#;K905w0Gf#1dHW4V0@Fa;U8n9dNyCTQk51S*1F zr6huF(po_}TCVI(t4?Q87ECK*FHBfnRP%&|1S2t9Yv8oZe2~ycCzNj*6SUaKz9oYc z>*JS{Nn7qQnt=<*38^E$DT~U2Sm=;}SV;VZ$rzi&d$ zk_eH$JDzV$$qk9W2{1Rkw9|!c851qEK-nsCIksOT*pJZ@{}NPp9_P74FiiZYF6C11 z`a&_mR>h|6C%BbXGb^_})$(n2K#MEuU}7uV?BX}o)&s5Ivd$2}u{^(&S}7PghNOE3qn|aR2m~Efrx)IF4HSg!|GQBdB*{VN#dZ zF;dFd2+t)PWGl19qkPvuc=j$IhQeT7(ivI_m6NNUMNL)Wc-VhcuH-ee_9$Cqj(Gpt z4S~mxE@8uOaj7YMspIwaEIL%~{Nt3{g>;P?uTR}qWKEocsK_Q3h$~<54i9{j?$?@L z?|z6}L~67tgUnsr@N^#rknnXm0s}74NoN|r@^lcn3;i5qZ` z;!ws%!(qompj_pwknkf*)<)43bg7I9d0~Saj(5cqBEzaz927UA9pgVaoS=Ou|D-5n zt>rG>iJWGH+Vab7G2{bs*v;L6j0kWM*VrbZDs5ZkHbUAc0(I@vf7esza2HfZABe7B zkJ<6A9-@ibTIjZSiu;WP7S!&atC{R|#MR}R`5)jGz;TC z{DHz`kj${{SRFqy$XDJdMrMuHYYL@JZS6piXIO~)k8nA1G&Vx2P9;WJ_Z^CcC^JC` z?Q6O|I;*}R2}X?37{BSl8)aU)6L2mD|NBfL*mVjrw=p@=4{IF-?V~-=ITl_Q{>3B1 z^ZynVH+tNiOH%_#ItNoGweu8CfjJq2jC>xb1G^lHJ3cYqg6J`(x9)PBuERnoJ2bFs^EB zg__y)7E2E_g^PHZayczShRlf`-nZ@-QXpDfSjE3Z@Q$^SgR!@Is^v(>O{&0$6W+U@ zJ18vba0Kt{l0&;n#f@2~_?A)`Glz17hHEJkT253MoI`$ugUt|f zR31ZD=hHnfd1#SLz04)ly(P%(wU&Eyk!*01PPg&@K_zX}J>QZJ$1`uPB6Z)+RFPg5 z_BeSD_l!l#h6T&2)k_RU?tXAi(8@#(V2!6-AcMLq(RjIUlIv=54VMSA;3B^84htYY zHgcZ1D#eI=b-bzp*A~HEfq&5m81L;de7oEE--cSH$dx97Y}S$DFFtZU2Dk<=tYkXu ze5v*V|Ap5HU;Rc`6qQlESG%Cu8N^tO2-?HP#04E`Oere9CEDta$rc&Yg zqwl}Ir7}iA`2VlENbg5f=R%^aT)7h1U$>44GGH6+0C zzd4B}1FvD2*~;b?nyI-95N^ncsD-I9T57Q*2Cesf#z@@cKb(X!GL)L~qfA_Wfn}hT zbn$*7<*D({=|NO`nX}N2OWbUsQjOleK5tsIKu}6=FPyoUo?T_sZuI~3FP&Dr@gLnd z`T7`RH}nhiE_cJ$hasaxIsZ#Tluy2Bh_e529tV;ao#DLV>ENUIl&9QEF(*FxA$;$B zXnnyTP7H|z_n*t3Mp*>B1?f_oLAiS0;a&*nLR;e$m7ek!Ur=I3?$A|rW$#REH;@K% zwwREJh)|sUhSGGP9nAl2hi|j526yU60&nm=g(H)|reSi=<`Y=sPD#6`6PEq8)z12z zgk(|)wOQl#QRK+Orn0H{xLM0PsafGw3n~#7PrTmiwhleX_+%iQ`*G=Sr>eICJtlS7`or?6c$w$ty4*xrQ@XE@gny` zL`7UjjuLI(-U`NGh^s~_s;x#UHm-@uYcY_hQQlp{fS>;{EiV-l>-Fq=St=!P-G?T8 zfLInUcP+k#6a7SbEa|KiGne04>bcl3mwp&Pi=+>4(0+lIE1c<#DYC~c*R#>d7;_9x z*P^_~=11)calp~nDLS<~*N9&4S2|ROCPa|VB)?wM)CtfQg{@75sxfi2nMjly8_rd| zG3^_20rHgn{uG6rnC1o7?k9rM?xZ2Ow8I4a+?F#QYi7(OGVq;s9_Xl#U1y+o6$ z2g+OJEd{K;r#4g{Tf0h?VxW~fl2;}~Q(!vp?^i=0SI2-HCgxL;YV+?mQwtB z#rRU{JTd+meNjl0b?8t&@7M853BQ_PPN{bek`V1NloxPRy)PEGmO#+BR^PWn2BUkL zN62zjjEP#`Y0L}QgW$)G+2B;8HZ4}BR}$x`aZ4Dk>!c{u-KK$5t7ZPOYOU7gb4f~@ z{!d3A&-Lw8S9m16?10$lYhpoIc45~_u1`= z-8GyT-&*vtP7{BLXy`y$;~y-b-Rh{IfSOaQRVULYE1K8u@xuvUf(fwvh#@(r=(H&R zuFCQ4i^@7r)fDXHs<;}E(Xl3#beYH!|4d{+uiJIzYFF@rE0+!Fpwy|8`(*TQ#wwo>uG0%1(7-ja@Hk9=u$>H#}ql0pBE< znBn#EGxDHvIo`ad->`QuE33AHE-Z%GzUIN!$=4b@abgR)XBR$M*QOa&uC+}l+k z=@*jBSjjS-P!km&bv@NS+a#EO1s@cTXn2Yi`v2nW9fK?h!fnyr?rGb$jcMDqZBN^_ zd)l^bYg*H`IcQ6s(~6p`^50s4f)E9ZFXTtGT$!c?3bLKF4-oMoU&G zsCp`Ro;QXu)jbd>@tHs{qt(8Oc!W(n^5Y|wsIFyQ5G~}o$ zV?IX9w^(tbSkyQO$An>6{d+`HnQ-V;D7D`jf~e}5C!pE_vT^+N37RVUSVC>%#U{nq z7JOn4G`_^KmaCtF?rqeuKq@Y7Rzq?F+!z;b6|E<}JvCE#MUh*QYzDq8v? zpo5IWYvd@gmST!5d*27vmi*9!YHRFqL|vIbM#qMR3(3W@!0=FtgRi3#ujD4QhVoMYi(z z?gx8nG~i}OD(z`5>&Cq_Y?z;iV&Xc1GeWG*i5^MYV8mP@*{q8o(|EG{SZhw(YL%f# zO5fV7SntHVsjgprS5X`)`c%0Md|8P-rAbKq+{JN|;8FN7(@^cc^H9b6R#@FNYYY~; zox58szCw7T#mmk|NqcY@cS@j3p!XMP@hwPHYINexD|ym4gOr(8aNjz8A2J_PLR276 zN0}J9Pm#Ccy;P4buk;i~;Yw+SFlv!9O6h8{bvz7x?W$T-uhkOvWQ-zDLzw2CfTRrF z{!CG&owwJbsxX)4~@W`qwIs4kqexu*IjX=gUsPPDI{;XT(J7rw_ zx^yl0yx!L(fAD8z#Y_7HYDZ11Q3M9zxcHy9+-<1_-ejY1C_;!?b=Uk}(RnnpIR_UM z(=gBOFvK>6rMBmTx=Hr^ge>Uk`!@0~YO6QxJu|jO4qda7mr^j(rabf!V6<&P{Aufy zUcuSic!x^C^}gZ_s^Dv5u3XRjh-{*;jxdj>8p3A(qnl);I9CFWneL4jpPH^=+c8 z+d8=t`}U?g21&9rbPpSf7Dg~#B!8{d51Kls>;`apeqD?*x@d*z#a*p>1xfk$>6B5e z_m;wUxtv7Kc^1w_6brRb+e(-f=qWZeMmk1uQbvB#A6kT?R4_Rc;QJen<<`(+JZ!=3zi=z{qLZu{$OWHl;k50j)eNSR5ca zyRcx0kn456F{J0uf<h4 zi60weWL*jnpkZE;IBaJ#XmKFlZLK|KI)>)jme6^`@n7PklON&zOU>_|3!4PNVg+s) zd;bW5-7-E1cD-m@3f*XFcDzPewvt!L&L!zY)UGi0HtS40v>*}ga~21~ttaJD zMpxN%kT9yPl+9Wo-htJjh0;VFyOca^nflvYngi@s>-><6R7BRETaBxG_I5Z$^fIm>dcrJqU8lI1UGAn6Rma8hnvMqTAhuAl42Q#RMwm z@rJ$|rAOjhIanH$RQIIZgTS7m=6Z31T@V$^QEGC}$(wqG*L%@XK3(Mf_xIkPphvj% z`^N{AW{cU}CeA!z?`3(;eEz(k*BC+HZNCT4GW0EU*G&QZEZfDHNrM|r8vEx)i{d?o zIC{(W{9StkMxj9ncusHU1swZ^NjXWXO3#RgIZz}^hmnCiL*e2kh!Q0 zJYi`0#8XT$d$tYD>~G$v*Ch#Zrp@k#ZL&Fs_(DBQ(9YrP z#*npMigjef&d!|R-tl^%!W$mEXdZ#>UNTHO)bxUc>UKdM#f*A(RJ$AOrzg|@-{Nc=MKTqt?Vp|KoNH*>s-WiGj2nz{>F#bZ<=n~IQ19XCTr_LN!oSI_w2vu z4wh#80Zd&OOkFaOJiX_u{McTdCe_(R(}X1O4bj`J!jKAoqkb#PylkY-?5@T3cX`Nr zpu`(`5Jy89Ux)N;hDgKAxv%XA(xM~VXQH zEruwf5}5konq>+0;KcO2$J}I<>WonBnn393qqg?EJd>hj=0i*W&coa5d-|N3b?Od> zLe2{A_DUHh3Wzv!K?fiM3NNN=m>1kz&~FnUm*9^C00dr>=}rAV7(>pcwSxx9X7tGC ztW;*uf2AHLo}zV0b7T^dKx2D{2)5LW6ngVqY``n!S+4FkK|<&pcMeQ|7ZKzOz!2fpK)3|-HiS>7Rf3p{&)>q-%%!bZyWIRtMLDJz)ByP zeMkQp^i6F7NUURvKsYn2@eA_$c4qc?gz=)PzRXBm;{OSDk`{le(olXK?Uv`ub-}94 zQ|1D8JXyzcLwRv945>OB1>b9YLISz=`ipjdpO4FqN`0*?IaCGw0U>|;SL@T=x5bCx zf%Ac3!rIjzbZoBt7*TZ!k)g+LE>A9>s9o{u*+3S7Cu|s{C!`W^o9OmPSyhbJ{)oe{ z{*`H)RUnY0ZrBmdQetluhy5~S@%*3q-E_<8X9$uCVS?y`LWq?0PHe1$7Om3186HEF z;vN4mrRYCorg{CQ&Beaml_@BeNekO|1E|D@@c|xB*=rMFBqT57*s*(c&IEH9en51o zkm7%Feea|qWhZ@BIpnzV7DbAY z1ByA!8W<+mOtB*O2m9&IP?}7VsgT563otBTME%FpcZL!5!)UFfHn@2^1}Ggu==b@) zibrca20W*~m`^=!=OUkJ{j5Ctx|O=23wNpBv7QgOy-q+?WxwP(F4lC@mHDZ$nYrU! z82&K;D779$jO2X`koiYSUg5)4GQx+0`DOpk6v`)Th&{(9?ZHote{~r=x2bnK7GhAt zNUOHpI7}(hh@U8g4^kiQg0vJx+9~kzj&Ruj=E$MFP>?`)g!|$ymB6n5sl_I=#K<&(Ht-yUN zFZV9btBp^JUz)nS?3I({+;{j>a`lCN3(iNp8(7+x-Q`hLQ&aHf`!rBfyZ^eZIJ0^> z2wmpkJFoJ+EZ*@ia8u~w`|sqn+5bac`wclIP*^Ca7wunRKmPv>?9t#$EluBPUnmWPM*Oh9)UA}@N?nJUnw0)RJ8Pr2aC^i@WCD%U4|D06;IaRI)wMc- z|3zI(tp5-1nMS2O_{R=7MEhUtfVaF~5ksPv4wlk+&>cX$2a@@}@M430u6mfFnCyP8 z-nux6=`M8S!A_C7oS>ekdZfi28pjhIUe^hO%&F^Ck~|lRy?)_0LtP5l5S7*@uRH8- zPyge=I(Gd}N`T-`sjvT(-4^KK5usJGZOXd`p4q}*CiD9!EM~X4@L4x8EBD^$7(Byp z$2>31#sQe=ZgwV?w= za$MMzV-?9yYK=Q&UXIC^Vq^IUvD09T;HkC$vs>$LNWESWhYvLNt?k9uio@twLLv1D zlkREwkdxNJk%Ytv>nrIGtPFR)t-?Wn7J4qP%VNKw`DL?!&hw(174#XE0PEKbcvI4PCvn@H~GwF zVrU{pp;-zTAx4$*=^QB-W*u=iY`!{&BI&G^j^cFllRGIf%VoAwqLa<8CLp94fi_7$ zu?t4ZQZkb;smQlyA=Z_zEk6@$Vp(P?0_bjw##beY5~5Zd5|3cFS8gB&_+OJe;$o-| z{T4|aJASK*YFQM>^i)7-|8#Jzm`m)C#FVo8xj`$VoSNGrNa@;uQ9glJJ1oN>`&@+5 zXWX4+u-dKPg{yl5>N<$z`cM=8yaJR4q}}Ebriitb{keT`QUZr3O~S2e=|-+Rj74J` zf3_+aYAVsBK8{qKUfjPQW{qe(IGmOOG0Wg|3S?tO>G{I`m@Jp_t&NxBQQoW2C6eV! zb3LeBP3aI+|J{<&XPWrkGfv?$QpzW+L|F9Vn;ub;r#NN=dG0*$PG?R*%|2=zuC(aW z!l}yKF&WpMqkJ2OtHSUqVGPmOfL@nNgXllgx>O++59QU78&)rA^68w1Fc`ttv8HPjql4U`Os&tEDf#pYih=1irB3fc{qAJiWR8m@jdIS|*zHoKUUlDM6 zzp>8#hagy1wo(e-{c8YG+HeK=>?R8Aof@w;)Y|S3VuKYo_mTPh_prb$k3j<|+VSxx z0h-@qRr+QTDx&zR1QCJ6mw5NlQpE^sOl1N6!l{z zNCiEq#5^ej?f{4A1Rzw1iu$sRc~mpS7!WaSvjC>t1h(MY-s2cBXhbCaH3(=%33%wb zOgtq02HJp|Au<`_ZUQ1+L61-gzyMeXX3_=Bz=C45Nh6i$JW1;Q7UB-_`2&5aIh9=@ z$<9)K?d&JFuBXxl)_f>9{oR-=?>18Z20=iZyH77MC~AVvDa0tf1pH`CCSHOL@cf(q z&d-?<2QvzvfJ?*)d{S+oTk0=IBnK^gLg`BWfTK&)$zVm<0B^t(;7W~ctrY~K{i%9) zo4TJbz#Di;n_%t0OQPTO9Z_--qJQq7*ouxjVlP3xQkG@sN-J1vwg_6b zAwmd8htTF*YaxD@<_AlL1MZg?L9W;sGXo%ZV3%efV9YF|<`Q!Zw8d3)iZDA?tOx4Kxc(Sqw;7 ziAx!-5}-T=A&Vw#EF$WFAaa<-8$&`R+z3lNaPke=xV0MfS>vlr*t0u z_lJ87rluUZv;xN>OnSfpUKYOtJ*x>&(L5*Av0Kr^cuA$^ zRj=i>&K&)5Wjlm-u$*0VLoDF!!^y#Y?7Jy&S!r?H$;{8g!-IlLiwo~;h%H=u&9M>qr;eaQ5}%A89r=ugzmgLLk|6Hgo-kxNrcRmDbaRcvUx*uO`=hAIcVN1<5O6kd8Fah8(b}(! z`)gkzsm-v&df5p^?fyhITG**d*e^GgjP18JF{8o$>D8e;1iRi#mE)A(OD^ zr5X&AOjmJ2>S0y4{vi_2dkE0N%cw8_u;DYrDdow$(#{S3Ft{G;fao{b6Q{k^q&InU zi%vd!0?F{QJA#=((df$7}Ub#J+Cuyu!6O5h+ns43n%v z9YrM3nw*`if?q7r`m`2MQQ4JMsjW3*MMN5)t2U~Zk|e}e<}~0tuC^8GCGUKIN62{g zGL}(@h#2uN+z#Q~m56Q?8MQ-M{@lX;K(byak2P-s8qh`I2mhZcvARhUkC=Ie_roO# zh$ulJoYM=R7+KUD_#uPGGw(LqK$}Nx!8d3z`7zP%^OlVYzx@!HBg9T4yB@&&!GAk)Tub* zFej`>Wz}@t9oF=?V9^;v+~AcaBO=DJB%#lsh+mobo>85SUyblkOba>gE;*VvmKg}D z)Tn`>fv^}+N#svaWe#N{$~PttW>!c8e5axUS%AI58B|oL5N3r@Gn>dw_;gFkPE@FB z>*Q3ZEn90ViWD&IhS3wEmj-^L1ddY(i>XoVkkVh1xR(~575E9Lu;iuzoYAq5J#=cZ z*x{W*=yt7rEK9lLB!4EOI27k9{SfTVvIw_|xJUl~!xH4hE%8{WcZ+qjZJ)Bl98||{;0o{B{6s9@e6)%| zl(LH7srXzM?_)=zyg~CE3PXfvz*r|sMw5JK z6IvBC-cBIi&hE&l1PhEJI`aaMWM|iCq9_KGcDrI<_SOETRfeiDY?690z0JzxQEzGJPmo04(6Uo@QH!T*23*e}6 z(Ik|H5y{EPrxsU?VZqNYwsOl|s*0mnRH*$BZX#p-2S5`4bh z-0ZB;U9x_y0GAEzbu$0eC#6woJ+&(>(pQR%Fz?W#{bO0)3+X=Wn~oDNQ=iX~8qCiY zGeeZwjuT7S*myuT^mk^8HL&bwS1J?N9@3BXhRGqHE{nHe`|T?w|l_m zLYV`gqbpz&q{9Ks39}CmU{b5(+wTDxfDAC%MFwDjFpZFT1F-Vs(7gp?##*i;8rFV} z0B|O}q4Jr1xdX%jv2QRqm{(V+U7SDP4jT!&JKx~uXF!UUGk_idb!W34Xp!K??V8qVwiwO7-PUi%3>Gv>M$b|Aj3wLP!ItCG(@4}1qF}+ zMuDRN2TeTdm`J9;(Lrb)-XNk;-+%`L;3xVJfP}Wf9a$7O4_0+`qDR3`?oa()LN3-u z{QC4`pw)s5X1JeW!JIE7UeO@v72%smydf#!6u!vI z`X4^ebpv1wIe-hijoeWMW?8NN&UK>$1t$=I7%Mzc*>oN860wvwz#} zm+#(}gJun|QRiI&@6$tdkQb>(!xT@@UnH8)=XlPTxJj+g;}n4)u*bMz>b_OO=c(?A z5Wyb?^Kj>Lic4*T#4wlY8b6)#pAjB!Lz4#%92(1+9-Cr>0fHdcN;y9u^_L}}9}oZn zfTacj0PkRXLJ#we0i)HA0NUeYr-v5-603Xm$aC<|dBrh!(&<}(BNLvHQNRF%aKJzz z?6{)_5*Si#@wOOI_Ye0r-Uq_;!3jtIC&nMs|6(JKaOtFAQ+J3YFwi6sxuoEjx&DKj zbc%3Et6I(QPSnPv11dLpGbDpLWg%0a#m)DEu^}bnuX3Cxb4aATt*mn;nHT;+5Y?8~ zIO8$-BdVwWCW3O&!DJQZ?Hcn&jbSYb6S{E94p}bLFCAoYuUl^9I@yiGuY1XS&B?XB zLw|-C!&G|hZjn`-`reVe9V3g*Zt5%THx=vQAiLRZ7PXl|8=pzRlFieF0h<3fHAA2Q zkK=U8xC>LjeL9dm!#ebl7R2bsUl895_iCB?fl^mayk0O zEa(#A1O8WMMX^7{`4P%;?2VbT9XrcYS3}2FM{S07T*YyNX|0>XShwRxnI=<8Z9_AA zb-J%UvxCtvw7y#FFxe|MUAEJmMdOd&uuxJDway==$A1`_y|!sf)?niY|p6V9<7 zNRW?UM^3Vz9KV{Ya&yi{o-ZygYz3K?>y7@(yX2hUoNfZEOYx%H{Cz250w4Pr-ZdSB zH)RlnoD*k& z2+;DBR7syS>1!?XvL; z_99)4)RG~zHbUCFzVs02Of zRhBApD4INppGu0{>;Tn31>I^|J}VZpW^URu%r4UCVMh+Ew5GxR?V2RF(f=NeWx>ILw*R$1Tw!*npA zn2$LJnPX!J;eiZn%KU5)jHFay)^-{Gt9dYs+#xQ$!qZOupk{g13~jqA>NnXtzPq3E za-f6FiPZ!_jDhhQ)Af>k0NGx9W?=N z{o^_CWF!cnb44f7_T#a?B=Ek=k^I_g(xGXFdf$eaOT z_~!(Gwi&%-Uc$&rXP^?=rZChN+VLiB;Gu!K!>=^FSct+Ut>K};dN*+OpHilC=l@;G zbORpEy1Rwi^1M_Mb5}NapCY29oV@rStWS-K{NA`zrCP{k-K95+O8x?bP8T{(L-3De z(ZtXvjGpbp8aXdy|(hvljY%qQ2*DeKJZ=P$GWP&GY; zV#a%Vt}d4i*=WVdW$Y_?&qun7?78nLT}(c8{DVM4PdYc3-Q($-o)Og4mKOBmzckdA z2K>FtJ{GqBH%U`2(2VK7@c^)0c{%%oYaCu5g++z7aWZj-{mD|8&T{mMnG+X&z5dT8 zgz6@;jE94i!{$70DdW76{x>|~l}FAG1BWU9lRx*!1ldM4b!QeI$fvGq^zRe1i*s5U zOK-k~Iqtu4t~6ImP4=3SDRrn7^nWt9P-FUKzaiA(nexD$s@N((7rITbH^_em^)~(7 z#~u=yl=3rGIV)mj|AwP7V=+p)DroYXEr-WiAwlXgD?0cZ+ijj~ti`3?xj%w`yW9Q~<6u%03TxnUuT4ze|2i(t?%(2OW}BzL=w z$V^t+YZH+|M{VgVp!OvJ!OTrg9u>UpI>9p95DbrNBVtD#Zt?KH@ZjzssXg7CAwNUr z{u}8k)2*oepP(o8e*=1=ojRznzX7%0_U?i9Ey2Kw7Us!(!*7>6gf1uyqwlI8j0m%v z7X2yjqDNxSo<@b*V-v>>-MZwld4q(zUW?`?#Xh@jy*KDC>DDa+l&!x5t(=x{dw7MV zU%XUiErz=L|4-<%GvJ@!AjO^mUI^j-f7lILkNsD>!6nzPAO{8JzW-u3nB_ski|hUL zfGhqD`JV;Ak1gO1H~+f@fT_R{knpAxuC#ueT)0NJn-NSmfXo>c#%-IL7ZRdD{&soo zH+r6@{}^_@8ZaPWah={1mr`RDQ}I-4Y7SfT zC%6+EBQXE%7}(Hm)*6#uN>DaDFrRnq7SN4N#PD8%Uk15T42Obf#DApL*Ss%l#TbV$ z52}wCilVn!iUln3?&Pcs?)f{5*xzxsEcCGs*B6G1x~-jDx96+LW@Ulp{7sUv=irC zD}5--$FGsHA9KxY{2mj|-0^EJ*a@hp{mWWJ@jwduaJ#AYoWKp16n}sba^*N(koIsq z6=woo_q>>!K4kG7Z*8^Zk;y$*%r|1wq`{qz<4m;AbdAyoiviI%Z^577NCds>JIaRg z-+)Ddf?-+QKIQ{0_P2Ln!6AP9k;C|8X8?--(Cd1@YbW>N3ToM?-5&vnKu+&#B*r=d zIY~iCB&Nn110N|r)osb&%khdrfHhz-_uzlB8dt?vQwHe)jzEE-($kalYk&B>XVb>x zG7oT|qb3k_Bz_8nTX)2E0cEjtH%*YDf6XSJeR_Wr`n>r0ej50`M(|QpfRTode9{nc z=;!?1Z~KB7fPK(9VU5Y*HB!wLGjx+?a1?ovEtu6+adu9Z)DuBMBJ?s zF4egFVkBl$uM_VD3e+J0iOT{eZV01ho`G`gxOEdH@Br(Npop%*Le2Y=^l|Ff=Gd+;?MR-R-7t%(%_s#7y0xoVxZF z=`oJ(l9QIF>O{yUkZ7j#U7RojO)rLacQ`2jdFtEVX3dOWQ|*kn!Nqd8!SdG)9TL5t zU-9Pka+yI5_c3$fS_8I!R~WVe3lzug)@f{v6U`^EgezUWk-yzfJ37CRz>XuU5PdMLnS~zTX zl-Js5%_K%uh30-dCL5Aef-12Ep(%zYvPR=M@Mc=r_4uk>T$+os*)*qr{5I8KBL zR@c;N=&%Aygh&pUF$uB6Z}EYHH)jdH`X&?~QRz%Cl>N`_{m(BhcI@f#Q}Mz05ch93 ze%aCW-?fMCKMv3O(Z^fIiq}!G$&xIS6#oQ@d^ zjMtyaN3qu{Ew?I}oBg)OX2waUCY>XBzUXOFRrhmE>;8JDBm2k~9mX7d29*j6Cw?(Gix^C=7CtW#K4Qe-iO)(yMX$=}@h6!F< zj7CY-+FN3V+4N!9x1 zf^>tkVmON zf}x} z+V(V1do%euhl*O(_tCzH`BV2t6YR^Ny_JIei+1np9LQJ}q8$Em=Qwc-{BppRp#JQ; zfdzXZO!&Gad@7lrKf zRdRBqLKkq0kE!evV1@E(*hB4Sa!XG`lKO~d)%FKo_EwkiniffiV3MRC5fe_SpRJt^ zg^t+Vb}FeUPsn%}d8(-metZKfqD~p(k|;&954fEKzRN($6pCJLs|BddkcP-TM`NU2 zc@ow9lzsn6%e_~Ox-Ki#hRB#9eXFHK<$<(a#$~}Ow|zxyZ($G9xa9q%pR`nCkKefD z$a4Gyul5UHs82QFj-)NX z*040p#zBfDn!)jDXG<8n^6R)+t$kKF&^9dQT{n230L~60Z!Q7`B;*d@(WJxA(Pnm1 z>09AX*IAU0A-arSj|Tmkewm}~>JsPul*GaX_= z{Y|%QX8xPbstoWF4IlvFlO_k@D?I@4@pFPe*OMAbpgM|y98e`e9#HWBxG-SAoamAv z4>SvHz(p~>#E0>H0Uv8f+16kXdtH@-z{b;su^Jv3l zQvw%rGjzulkr>hQCFE5Fy{hEn)bnz@#70`=o;<5RzWA3C)NJXhc?(u>aaap#|rqh3BVgV?l}J@E)Lw>xE%{sJa30s z`WW&5=-ueU(@nEN^=eM+mwhg0Ng|#O{D=!4n#W8(!-@1Dmga!b)6#lLM^6??jr=nj zX5l77V+j3vkb?H-Z6aEqfFFJ%0JK^Qq!n%A{9uU!qD_YhOO+(Gt!p4;j+EQY)^4Go zj#9KSQMN=fV@6==I1dB%_wwqO06j(tl7a1Y()^ETl<Rho9T*1uE9XZko>^|3JF`qXL8CZn$=T8+A@priIvkE{X_`qAr zy-<2I)LUN-r{X6@I)>Jyag30N_KScBj{+vqA=naPlHSLN_RZIc7f@5 zkMBc+DtdzEI^=u1YEfY|oGtXLoKCSqUr)m-=49E_kx0cZ*EM_c10U_<2Auuq#_kjF z3MT_A%8qQ@CkXd+vFEG&wF5iGWFp< zdsNN}=5AGYtNS1_2+4^-5ifK>ehfCTsT(SNpjEYuv!!?N??UHi=ra_S;%- zq3~Q>su}BKKa~>llcYT4vhZPj+N0R0D3C8JOX+CR+;aVPT62syJ!x_aQdbDCo1y=` ziMCq#BUd+tBor)GQ;Sp@g2=r9xyv0j=nX^_^-`Q&CiK<%xEfp!`ebnYRgoqRq80|t zc_T`T#u)tD=U46;8liMhUo`#9$|FlBK~SHt5EWDcz9NDm72F{pbY2mmKFKMdS_xzv z&BdNzN0Up=rOMUftqs`u^byw?A20FAbAcHtdXp_(fh1>DinD17Wmh;9 zkwrcQoWd)e_|D&3{NUS%Ig4yJ`e^Qb0qs0PTPk`0Aai=}N-e4r1W}GkFYE#^W7 z96bXxl&cbo=qgD1 z6o1wfq9Mn8v|0L87t}?-l4&$y#L_!hwYV2Ekv#aGD3Lte8~EN(rw$DgrTO-J!3wrr z>7->mRuDBD zE)ly9@PKS$1)NcAH?e6z^(YsDt-=^K!#%JSWEA^u7}%+UG>Y9=5|~YVta;3^ze!!# zCQAOwCsfx^;+RG6I0Kbw;zGpL#BOTsacRqF!cm|7#L&vCQfA0)$jx4AyIOkMJ)BJB zjR(h#;}z_Q2M0$f1N?@{;=18^n6eEg3k-P-dDzVzRzoj~q3~Vy*ZJ8Xz{b%8kiaTQ zGep=S3F`}lffG@H)5vYR5SBm5q2P0D)*7)s+R);}!!6ihHw|*wdYHMmXJK*)=tUY#rC)7Y`kYT&t?jHU)*?C}+#C-MueJMw;ZJknPraAh^ zH|dswcd2e1?h><~+-*lmu7;FWUn^V_mVnfz$XpbhpwS7}Sx&Ly6@?sE2Q#a(CAYOo zgIxdDwXv+l@WxiH{W9K6w034udmWl3B@~I1Tz-hBSeTj+hSFl}NTno_!k|RUBP>{> z?Q64icNKtMbtog)f}>h>7!9#x?P(3ZWSvs0&NG3EJZue-S+8BA4T16M20iGS1oyJS z05!4_P;-2*aQ^~KgB#c%zZ9ayD!vh(9~tlt7hl+(=L1WYH@k_Mz2;N|NUDtS_+O~`^OBIM%hKo$rGm~E;wsI7dI z8WzWdO$dM!%zk?a=j2S)H6XX?8e!IX^$l{i3G4)W)){@a>3uKY6eV`tq?5pqyWgdP z0k8m4ib~=gaLOWm2eJrKzCl#36DyAWL+^15q!!j2N*Ai#-oSBde$A^U3erztV`Xst zgn$2p|Bq$=9B{M*SR4j$bkV)kZ-6{l_Vyl*Riqv1Zq{{t)$qTsgMiY&1J;POUzmO zN?5_J0Jl&{xmkPa{465cS`fv_;4W8hjQ;Lhj__E|9R)paj*Q+6r7$0^8;H#;6!1-2 zo`9teK`cdw0vCLx>woj`tFNdvniBlu8J4bVf~UeNpn$3O2gof%1Y>chhp-Q^!W>a= z{nRsP9vkLrf%V&2@}PEqG232+m`VaKpElp}PZ<_MaGt2`H|sfscz&GojMPTQFp%=S zQy!Gu9MR{;ryc*Vp4$b5*eu+?@&39QM=2WC_UDGukv26>TY!C8S2c#qB=@Q7FCqX#O)KZgue8UK2a&Djz#6h_>Tyqp3)g56pLj?%{mVI2tfkJF zWRFVyDDKy9-Hez*WlW50>eGE&)35uD$5&Wl^GolkV`B*0M$~GM4s4ife3l^LHTp2$*VE+><1qV%9eV z%k1#a)Ut*zG|r^1_yAqn6j-7|zo<*MCP-MI(@E`UWBd~!*R<#~V5H%$f6wwQS?3Ow zeWPex9#bc`?$Ion9}XD_F-m#}scBhTkDjc=dwF-eAML9tbJhi18t+>Z7`_V*DWg2Pl6J>#! z2^e4{rm4ei6)7u_ai(g<8p-!iljmi?_c+=m%~EF^UVTja*h%ZwYTd~vQ!{>_ydf(H zq~T0A{>xD3#D*o4Jb^q{b9f9C9cOtrO6QI~}{&zT}} zmx{nUpIm6nd+PahmB|m)fOcc5x|Fnp&4k@ovqk^AV*U%>smY?L!U;Pi1Az%FC!ZcE zjawGZV<{_P0hm|~>cStNn!@Fl<7)JD?Ph7+ zU5%3BrnYgi(yAW13+hQ*`^ywHXV(sKo;dzYHrBGO2b#(}((0YMR!9URZ0qm^k;#Cr zEtK&-LSbywoc+X9m6vo3cD^6{XJ=^9zu1 zpI$cd`BAJHI4{)ch;3NUWW(ZUrX>N9>>%xvb&ect@)Ns+tw`;Yms!y zZmHerqMNJDJW6}K2mS4?_8ea8oR!;a1)0vovgAE-miRLJ`lZ`j^^V1tL)*sKmwV@j zec4mtf&dR;GgQvD;Z5NdKHPc_W_eF6CeuBYn?ZA)Z_VGqrGp#v?uUmt#32%fQc;ml z$-vLn_t38q{|x0b4~?4CXu?d?9C?eP3<=$(R;mU|m6j>>5Cz=UeRu0FdJc;SKg-={ zxM2uV1k7L8R5#cxQsoJCRE*Q%U8D5QTFPZ4Cx$wgP$-oZwheXZYN*w(<9h82 zMuNK1(|WOH8ZeOwTpnP)`A251to|b1nyrxUF1t^nN1Qcf6?5v_bZUAd zvRs+6GMAY#GR?O*=b)AHN=nFd?6BTe`m^=0kYkw<3ygS92}vdhIJ6Z%&0HBUBjXZi z)M$6pQB=TyD3R|2s4DKMC`WRf#&O=hU4y>sM@FcU z#7Yqdb(S>g=Q5TT(Cp=Fe2Zg7Ul9M<=&->?_Le|gT7zI*ZNUuFdhkP&+UJG&7LFlB zt>DY=q9L2h1Ajd7Z|`H^M0HF&;~78vpDBZ=x2tr-T+Q3Y3n9HFtF6~?H+f0=%FqQU z-s0w*!cA-jnHv7mNQbE9to=CJjr`hH+kmOetnJR=TGW%3>Tu}$E>t8j=$xQv~M&tAkJFyB3(>G_Z_!7fo(^NdMw2&Z{jw)vx=w zrfX<{@G@L{6x0hEJO)g2f-81Y@j!{|1=_4QzUJldN78|LCvygl{aC{Xa~Ljc3=`Ks zD1m)5-UO2r&XzHOviG_bRP|E}6O8671?^k;ds16-Mo3TRfyX6sj-eX$-}qtouBgiX zbngF|<~LkSheW{z`%D$k3NzZCo&cVm1;IxP^Uf4c`$n{C4%rF?{v~64gqX(K=38@7G+cb zB%TmQ@*BXdk~TsF@{#Betda)s6*gWVQ!v{-Aw!b|aiGU-+$ym9)Nk=@Wy0|Zzoc55 z^?6yhu@WUFv~gV|tSQKjc`&1J`&q6@4xh0ldEsB`fWv!`Tce?xT-C)9<3&UsN{PtY zo-;SJZ-m#UcPC|r4M!Ck`$j|wsnRV$D9Wgk(5RR6Lo3P*@l+tTi2fSwUAz%wJ$vfe zF7CZR!8@>9(+w~z{Fk?vhJmBjzhfZw znEw?6nfyl#bWGY7uX*SykIWB@f!4bJGX|;&b9~A~Nuy2bMjeStz2p_nQnhCqKGDIDb;g`5P3 z8+)DAG+ZSt+I%+FBQ&m@@rw#6JUzzc3m)mG? zdd97}Smh<%FRfE5%#qc58-K##Qr6Grj_2y}u*~cwaJ=KMZC#GvnJh3p$Z~9@0)o>eg(HOK^903r>IpcXxLu zxVyVsaDux#1b26b;O_3a@6Pv~(|t~NU;B|&A+?KIwN?t|nB#fg=?#P|J0M^)i4Q5I>a1r@XU!COhD;>Y{ zxv`V?#b3RDAklo7sCRlZx$|oC2}wzk;IeGN;abQsZ|>NN?KZ6ZHzoz9by?F&1e0B# z%NwtC^8ml#oe-Q})YU%M1^n-mXMN$9b(bMP%HsFf$u?_eCgRM4XYleh=_IWo57iLm3+zJ>c=oqbRq^TTrx5S^>ON2A zuGlnSjG!@$0Um*iA@`MEiBecG7MsCWZ!lk5&0T{pzKXfF7U}^1JK&!F$gXiTMvd@J zw091YPizbrh0mOj-{?ZLozvY7#jd^fmy?w`0ZiG!f8k~`fTzp=tY;ZKjD70_#> z1av;d^(oG1#J($i*_;?OdS_L{{nW+MM4wToU+U`2mQ7{>h>^*cwjx_Z^%iI?M2XDf zy-ezXzl?|;5d5l}e3Zbx%ky-uYSE`lL{SyIK6HcN&Ym!nm-2xPU{*Y_)Y{YZJ+~TH>|%AlYveUYQ<@ zfkvWav!`w)Et;&~z2_MYX;%Ivm|+&T1NyH;Ry}1_n|oD~Z6y^=&T|%8%nKe$g=U9b zK|}-U(b-YH373qF9@bCf*WeE=zZZNC42Xc~I%S)vd6dS%YJ%@@8F!ULqKUfqhe+Sp zf_Z(xRTf>H$!xujqBfTe13=`$6dhzSN|Q+)c#IPfP_rL%)p1cQDvaLvWp z#`lS&EQFt6_C($%C=_8qRfe?7=hl+s`9s$vt`E^74h@lO`|rgKl^1Yv^YaS0xLGYe zw^!#zvR-1R*U8hmFdT&ul;_rh513sEcx~|1Bcbw>>Yk%IPyi4rBMLMD0nw;3B3N6{$xHw9sM^uVYuLf9?W;?tfN;~B>0Km zuE>a=JU96{sXM<)up43*l9P}sQzM|{W(o&6yz>MFpWtEAMtr(|V4{T>iKr>(?GM&h z^VSFiUsdTEFY{SG;r~*!{AjIp(pLE4@@R8y4f!y~2wb!|mTvpJw6S6>?{SI&H?yHQ z=BC<=d#&`T-ogG_yj>J@y)#2K&HVLQf9t#3&p0?TasD8fKWI8HG4rePz!z9>mTOdI ziL91C;WZ^!lljOxZ#kZZJ({1UrCaZoao0OrrNU}DsN=QUtYq$8RKU=}b)S_ds`>HaM`c5j zY6VgVnna60Z5(xU2+MB`G7q&E32G+Xw3UmD6Pi^)PXCQKdHuf9z$W<#o7Z3wRCuI3 z#IX-MWyGb3kazha(?%wQFt=}pjACODTPjN|wYA?UFT+jniHN^ld*YYfLRd=6=u)T^ z)~^%+GCv=##UW9lOFm|ut;Y1DC%(&#eFQR3j6mp}Ok9zL74pg$0sXKt;6hT_`&8ln zS6^s%D{wgn#FlqX+);$X%7wH$jI{|kU6UdG9jSo7DP0|6*h6;J0vrr(!5LV7KMJ+F7XZx z@Wn-fl4i9_nL>bV2oe*^z41#QzP_YQ%0l58uq~-)qB3Pz0pDK+QMXkA))-TUE7CG( z`3NTPn><6SN8cIpC&2)h-r+DMsP$h#%lhvWXv&03$Ukk2`Xt=XM|s54k;pHBz8b#s zC}#Fl`m$o>K)IS$pqG*0->8$<>4_=a1G1+ALCPSKp}mecXJq z|03wyMlb)-{j(ja-1`RGR=-^QcRrg$pf4cOuZdv$1H zt6^LUqsnV)fa*qzr$YFK(zX*00(O88t*Vq2CJ1&={|%Hl0wgcYie1#1HRSUsMmkCG9e@=w&IhEi-$SvXVjg#4p`T(Z7s759 zY8xZsIxcA9e8<>vI|h>n39kHe+_&s2PiWI>q^*dY3`?{&QUpL9#u_y_3){G#f>iH-c3_y+}#xdjIc>+dzE zCvIydV(>g97PK?M}O__J$au*P3NNe5}$>hJRUd z0~-hRN6PO$w*v&3$qo0c3IiByLiQ?Vz(~FSX(y0fB|izSNw{=383gwtyVE9C8SP2< z*oXO%<0i#g-4WM;#z!k-A+Puihai_q80Ra?8>OYS6Z+j~k9I^@CBj+wQJS`k z!dRbKn(w~PH=6x8&d|7Ef&q<@W`?l~>HrVt4f^uH-d^j}q#MqDS#rJuE5P>=fO{nR zSW+Hq`oV+WRm(ol|6z>uj?Y%E#cCcN3{S%l zc^SqtbQ}SAhqmTIKK9+S;iCJ@4urTMQSjI}Om)hrOb&__xM_GMTi2>C$(t178z;fh5}z5#Sfg1YPxf!%|No>XPPu zE9hI=NxEo-6-@9V%PqW~R0F3LEq?9`SyRn(0`Lr1>^-;~gwZc~HC>HfISJuzx$Qmq z*c4vg)E`;FAbNOi_sJwBYBt*IOmU`s>V@6@HtX*vb)`KKnlBlA{JC8|T9Qgqg9-up z<6;O6o9Ss}JIO7PX@x^^bE*@CoRK6}Fb{Q04>!1HA|A{mR{@**C*KUjPxChAtdKb&pU%2cw#CQ-m9n&)W|4p?n>xzXsbsS9;lqez5=i|LruK{+~Jx#s71s0VkSi8y&#C zgqk1Kdr_8?!QVA7VJq9R@{ah-cH(x>A7|E=pPCTDpWZx5+V87T!(a-HgLYu}6&dY< z{%f9FFW+ldAoMU>NjFOMo$j0JfDWA2J|nY``9b{S8|I$S7i`B5^BTHHIB;yOI?y(8A~_po({>8@7gC2lV~qKjJIzX4erg=Bl{8I>soG$MidG)lxed;>aYvTG z8AcAC{VpZuAYo3~VG3yuu#mSCs?Qr|?%5>6`LWn7L>n=oGfR)7gU*df) zRljAp+G<3Oe*WCqot7(>ZXbfK@~%KL+iQ;{1jse8^4P&WcWLupysQT_ub$5XUd=iL z03yY{F1+SQ+#NkzB5j^N8@65i=hp3hX=1gndGBYIg+x`9q1CaK)v-RrnD)#RxHsNh zFIpE^A9z^le>EP9!o_ndC8fVj=?JqU(P;@9EoGfNoQ%lEio`PwEd}Fj*zv*3_ZW}F z;8vCU;b2$nnt5{^PoT9y|3;e86XJT?kqKpu?WjkPYi>Baqv3zKle<$Ww*q6sC|DfTqjA^QYNx%P2eeQ2**kC0o$wR*r97=- z%tCsK=8pDZ6&=n+1@w@e&5|P~Jm+P)VVp7b`5{}`Y@AJ-lD$tbY*W}hAeXkx1yGf8Oc&#cqt6N7wOBbLW!Bn zXEBN)l>SqYt0KZOn}y%gT5G=XU4m;UJ<2j+4332#Kqv%LL|q(V$@f%z$lN49!?6XW z!fh>zfXm|VlS);}a~P>4Fh`>~IKuh$d@jy`^^L3dM55&L5$o6DZ5DHW%Hfmvr&}jE zq2FU%#ii))AU$k1`KPt7hyPkW=w1VNV2?rQ^^1i-0+T$X#ILu(58&X~KDhY?v zWc5Q3bfou1;P$Y~O!i=j2Lbyv;4u33Cfw2Bc_FlA z*iMPN#Q*9&$gC0eD|XqW)i5!l+-5>3nA;UGQ>I^fwu$QsuDf&jQMv$ddmuDsL@__F zI)k`9wl3od4vR))w0U%A_d98&7?*9fZbQw!5ky;sE>yVnG zTz*5kdn#ui<5(Zu`ke5E)VF&Rd)lpzo#VxRV#0xsFQ^*Sx71TXn_yGqDBg*AUz{|KVDU{XKF*hipIXQ%c$)=l{Y7n7j#mLrRhtiwmU0p z{_*5nzt)E^VQZ6&QgKnW>=xt6IpfO~Bkc|&{QCUkh}PSfFX0u2=N=u@799r^lh>_m zdXt3osyF_J@c5;UU{x(Q1ui4bNUh5@b)ARDW!GJ=tji9IkKq?sb>rB#0i37E%t?lz z^%OAD`+ICZ$%iS&pMi%Z0f*F}5+|o$K(Z^=Eo<1(b$R-Ib;~d!vN4kc6vXlnspa*q zTRsG82BYuTbm#M>r_cegbtz{3lh#_=S>GKmoohWW7pm%jXDwYwsjInPV(iPjex`TQ zQ5&0-b~f!x3#IB*Qn02YHoECRDVLnOvCJ29!k;RgypbO_IS~(_f zDJ0F6h&B`p;2Fdk?$+D=&SZJUmY`P~@&jA+1#aR;4{Nj)2V|*#n?QZGd-K{dM;-4) zv9C#R%Q45QH2E?_#FBH0fnU~JG&hCc-QvOXaO%v%@lvLX^InHAB1V%7tIBzC(^4UfbSBruPs^ZSj@2fy{GiZRgcT1lO@X? z%syg`Z0-Q@Wp9_Q(_081>I;%D`CZKP56uC5_PjcgVxC8tw{n$DX4tOzl@7gtX`|z5 zqcw_oI*wYwJ7y>J1ggzfY~!@hYAtOGg!o}$Q~r)DpJ09CjiJyu@*r{&@|OF!J=59& zRl}J51eq^5cG|B}+9c?W_r<=7oVC)Lo0LZ37*B%i;pibhYG%kHkYteAE1+3og~Hf` z9}V;qzPHcuAv_I0cKpiOdaW5JyRcMT$ zE(xFkHu*eEe@4wfAU$f|ZvWx*%+zcO<)N-FWtcbS zADplxWDiE@fs$fML6K2z;C%gfObRY^QQNU-FU%jjqDw=|&jHXdM~{*Yqy5-VPtt5m zhk_>^mLHihfIhxFKnM1F*6h-6tiYaU^|mt|7tu_Y*Y+hs*VmNy_%g%$v)<;>ZqNp- zY|I6twerPU(T^^>C^=S;gImCT>*2dg0-$>7q6xYi&>U?<@t3m#PRs-X2EO$P-mkVC z6q?u$hO`H)LATB5HvP|7CN{5QK&RbU^X|RS!nyux&a>|fXZmwNlQ)2}!FcIOijvGop zum+Ns%npZQ4EecPKIjl7u!Kz!fqFDBubm2GpXc?q!EsZ!y;J?=V@Df)z?KOc3U%ze zkXNj8A7J?A_h~91G-4=1?OVtTX{kFsF}1tw#4heB0AEW=?c+K>=i{74Qu(0X{T zi*y2Lvae7E0C&@sKLqxPls_uvpMhV@zX?p}J;cE-_{g8Dh3@aZD8FWs4lTYp##9F9 zR$W)dr?(V?u()F7CNih=Eok8-DKG*{aetHDf_|h0;A8W#hHT{k%^XwLsFzPZ$7nExxonTchQ9n3iNdX z^7LwnqjdPmhuu3n%@CpTGIOdIoWFA8F zJ|DIubgE7^Hl$&Bq)XWM4yu%RR=cgg;%WU*ZpVsC==t1r%Dt!nux)b2vEYqD|WJ>`;GxE-#0mcRPvE`fOUY(QoK+ zFz8YGEBp<=-bmpnP`m;-WJ)M)&~A)4=61?(uw~zOG6@*uSO53nZLSx)M9z2SH%oh5 zXY^}mVodokAJ{NZl6NKa4l>O zQuv_+FWZku!JT7Ev1aOkJ?gLo#z+W*+xh#tYP=?C;gy*;Y9)tSuaz1!!n7$5KY4n> zda+Cwe{z~769^~;R0Pv6J9Hq=u3J-=3RH6Ze#8&w6YXP=r?6bvpy6^TTGrynP3;5k z1y2ObBgg_N38pjM4hgL)iY!f^6c1+H?rNh7)R}0U(FBH*;P@n#20lsmWDYEu%4cOs z;7h+erOth8oDEc5$zDO&wG;?eO;)E63?v<43BjwRC#h{nWN2#`y_;`NQcoLg_U$wc z=3f7X)0wVP@G0D+suv6Gv!BV+&DO9hWd&WQOOQHYU*&#womDh^J#FQviu$qK%K3QL z%hGUD9&NSK(4y>j@RW&4VXE&6NVQh=p258K_=2U>aE*<^ zX$fS1AoX3(b5fCtQ|{y zegw6@;jGw3IR;$s6cHrp5*bUVBsQ-e9gXGfwxGjC5Le>CkFf-u+h>zWegetCPOZ+< zk;_viU2_ey$<6N{#yI=0j>n8+H>lYJrufEd7Y>MmojJpj!>AGaG{)+_Tzo`R#{8pq z^v4=XC(tPVk|GchU|{a%AYfp~$kaHM;32GV}Nn+1+DTZU{ZT2DGG zt*vc8)Jb-9xkT8iMV6q^aLHbWXpK@5MOEdQlDF2N^kjH(2(dI<+!7!88}+pSH9{zJ zG|d7d5s$Oud2vb($<}OlAD%S;6|P*wySEC> zDP^DRt^$E>vD_ zF^*ZUU%#kp3jIxDkc--HoJGB4#!m4c9Ld{dR~aE>M^~$gvKf>|SG9*8txx;c4rs(j4)dBC=#_HVi@76JuJvzjq>2C*A9mIM!#jvgJf(AtMEF#;B-98KkQ%sKHR_WGWh z9LZ|L;3h^eqk}gCIac&q$fJXGg}GMrKrO1u``@*w;I~BjYw(fQ3Il`{OEXW6QUCpR z*V_FwZpb;%xl()P6%Lt0h9ZcSUH$2qX}s}WeSJ<1AmzvDTtU6+zYSr82)PQnT57Me z^4W@t=O_nn=^>W}TfPWr4#*RXjh8V$`5|Bcp9z9D5(jw;dZM{tIAx-Fh__(4={f_3 zBpa*k_jii)p^foxrk=8dVj8uWxkyc(U!1spZgUNJ#Y)DLe9b0XSBy>*Raa}03SVu{ zhz=7lzeV!AW90}BNH?$jr6L-Q^liLAaqi-rYPreLhgS)`zf+xI8IqRXqPPWuUeK6`s%MxXV-Fjg`BGIci<5O7LmF z3HVBE+U^}z>&PJH*B6^FI4%*@x6d`QK+rb?x2-$X<=rxh&E(y*j(bY@DQ)RZ5(yQ@oW z?k@L2_G1z^#hri|!XaxRlRAepY`wx(6bhMx3c|6&FnD>&X#?d)?wWb(!7a;iJgv4* zr7$E%ap*xgfoFa>LjFt;1VD2M+X*QCN6^Gf?pg& z5V=l572th>rBp1N z;g6c-TVsw~kF7eAH^95H&rc09y|eQXxQyt9@sNe_h<_6&1(^#eR*cR-EeoLrAhtvk zAH}T?$u6$WAbe2>`$i5)azp%;_(q->K0XxK;))XPVL_AsG*>2seFT98N0oI36~~UN zL0*9-hu7)_N1wEXkOw{xoysn7++;W~+++g35MbwufCcPa5g37;E4Nq76r*}9U&6Pp zol+&koz1Y*>*ABUUA|c{A0%9tOG7MW>MVLsxPff07R=m!%hVUMb;Sa;f_^7>X=ls2 zy<7~T$C7!t5gCdQGsghDL@!g!yTL{o!D8d&jOM{ND@m^;>PVLTEsnm}po7vi{fL>i zb|w*~80uh91fVWbIgUffa=`ePF2drfbLWDd0-QFk*i`&S7wPnsuWN-7y;=rmfR)^^ z99$X-xz2dfT#+?pC!i_BDL?Kj>nY{;_iA7lm3au#1s=!7++;U5$DU`{Txwv;T~}M~ zF5_q_ucM!E&|f3AY>o;S3!j%ekjIlvX6OdNfT;t>PWENl9R*wWwnT?el#F8j)K(r~ z1NM&jrHgCd^zrMSQps)_*;U2m-XNL+V2{jK*Q()AwZ7o6Av12lU!nf`0_st$~ zV_fB}Ff|WGpYolm;L4knR1&t^VNV~p8-KhgE?>6gq=Dev2!<7WxH4f-tj#0S4d5uJ z*gfq0Z~P5m!8+aIJ>n_p=aKPsv26seYsU0W@e%F;69E%{`j!j?HgBXe4$o537f362 zf~KFGXd<ocz-%dHgRtDB!o% zx@Ou`iXbX3&Tg83ZEsk=CrK=-RDgo;ZMMNr-g!#2EuWcJjWui74 z{7#m&R%@nYK)EH?m1=KLMeq9Vl%DBxdeLk7tB1}U;$p!v=;POOm<(CFXxq9?_WkP_ z;N0cwGeEa(mFKea=x%BfIIq9>3wFEtZ5pC)45HH5)57j52SOQs`$i1ObzP}TC@TX}pKR{7yWRRr)t!!P75o`ezOZ56& zwBUKg_!_PTVDz$fCvz&ZkVFeMtzF-Izh^!iwXdo|gU_}c#eDr`VkkufJe#I$1xDpI zB+6OJ>~rk*&6((E`V3O%h)P(2)&)EGiiEBlA_n6&TzizpK%!&i$7%I1Fu12xV)0Tn zA%jf@%Sf&40wFf*Is}lhqeQIlR%O=dtWw3Xl{C!bA>0F>?=|Ns3QP>wEyB@CscdV! zelDVyyNc;G%NzAneiSl3WSK044Iut8OO}HF)hx*l`r9md+fbAfkVO3#vqTT4-(>D^ zsHr>wZwLFqXirRTx>ju5bLWK@wGDy>a69UXIpLCDDV}An@zKSYkuothSg74P9q)@J z8XsGeAL;TxcQo~XIvU;)!{uPC?;$rI&Tfv2v-ZxBG4VG^OHZRp)Ap&Jt@Elg4c$?aLnqk*a%JP50_*Pr4ALE@&6euS-) z2~VJD9$we}Jak`z5#4EtKAJ}kq;8ocl1C4|Zn+*ZJ7XY=B5IT(oVj@@U!G&2#9#a7 za6Ef9MFLC7+x7U%$LZtx$NpBUbjEZ(h9z6Xa|uhSvy#qS*nP|)OG>97-<61v;fQE1 z9UUD}eEZ8L^O^ zmW_~|h2v<(to}UGTFH&^6T+2HIWq@1tGS=Bj8uhY%H=%wclmPn=cxi%e!ANSI?HhR z7#f#Gt$ntm(}haZalG_0);1MPjY!Y%)zwPHZzhA_L>b}NID+rI&d+HG^XZkEmM-rH zYIo&)s)8aKipaF1#zV<;qZV3VknASFMUqm}eR>^`=&Y!^*MXqc1P75-QwJA!y$)jc z`^FQ+i6M zSCU&2!(D6}t!XLE?t$T- z_*g4id9!RBr(jA$-En41yF4*tStT<1Roe~o1DHDwZoJ>eO|ZzK3K}@;XAJP9p9_`= zRwEZ_X+KSsRDJx_LV&H7FB`7@@oP&}1@bU}eyyg6yXskJ+|}#8T?uNRK9$Bp5-0On zmH7&!Hfgx&LfzRpVvxT~{^LrJM0j$j^|I$zRLWW39`gb$xM--eKj?|#e@9Pvb$r=F zEvD#O$q38e^1?8b@-&jA!YNz#6~+Jy zq$;1zC5MDM!tS{I82fVOO-we{z9B>`1OvxTPCspi%Rj#*)1HDx+2~ZP`x8ATr|r4{q~m!RLTL&Fwehr zMVI=nY;%d{i^dR3WEjyye%}dxFOk`lu=ZFi0ckwGuasdVFFD_b<+tZXFFuR}6|in1 zMxH+y1Lvbbo(}-q!{?WO4dP@FRG^+AFFNO~JNo$_c&UIib7u8YbtS>};sRIpH@M&# zOpDCqh?el~vo(V?_v|5HYDi=7_d2H3nkGkmwBQvi1+5^T)+8%flxYTG(0^Ua`hk5} zFX*M1G#VJb%()4{W|8j*Y6MJ&i5{7{Y^FonzatUSY7i55!Uq}xlJe22qz@6&rpmC z(Yf{WVU7zll(D4kPV20=RV{rGYj9^&OBQn2zlZ(2*b>ZV3mAQqDCHr{B*8`OqDLPs zwX6~MUx7bDI=jkmy!Bxx|3u%;Y_$^j{a2XL$J>D%FFs)OeXK9c_2N5JH$q%wca1n= zPzd*to3^z2zGh0oMJMI8`2+h@YYgyvMfX`spX*Y`%DqUo$En-NLVKx8X3jm5M&N-H z+llQC>Xs85TQ(UOY-JI~UNZ@>Wtk4-YY1vGxl^9i^rr>BRGkYcG3ET(8Op&`eN!{O z0>z^Ra=dtDWYFE1)-VuaPGrVgZ3kq>C8(p@S6o?Po02H!}hT zipnv?MY{OftZ_9yt|Rz+Ne99`Txg73kQ&6A7#S@w3U>ra1Oh~)1b=rw_dSl^e3(HF zfzZp)%R>g3JJoDn=3dc|;}GNI%`+buSJfO*ewlum{OsKP{7|kD+_ojgd4y8 z#t-~R4s3Z+iV&heY=`?WmcZ3@Fa9K#TbtNV4oZkjG6l4LrLN@s{480A3R zfefwChNE9Qh&zy>-Gdya6Mg`_5j17e-`E%Rb9W7QS2LsqpU&`h1@Y+H0KEe{T34HH z15%mgr|!Chs`HPW?RKEsz=LH9TK%rTg0O7iKEwU@1}!Ck;%Ex(f*XXJjaKZ6JZG-VVGW%Nq)42=&?& z*ShNMa4*;}Iess&9W)k+Fgfx87lLdM4r3U3em~N)winO{s+p9>BH?SvQxPAH)Zr^%!sr-CQqCwtOLEj}^8lXXbqCNGx z5|Skg`>ITG5vS`c_GL|yrd7v`jhp&fLXsAnDyP6*1(zI^??t=&j6$k z_%OPetWc25Ijd1}wrO(=Mrgl*7`70D7_NZ&0h2z6JuvBmUeB&*Ru3LNy#=T|la1Wc zWpPtbenkHw^evO+`ylh)d(1I-{OlqSyfaeQEdMt0!xBZZJ*3okkwL^mL{fUFTe+E1 zicpi;DOR#0ZcW)Rj)SUGV8qN+z*c*e+)2{3WyNxQs;FNLwt-qH#+LO>+&t@VY%o$8 zVmC|?drsbJXleB4Hi^+k-WwO$1)1ZLh^?6Y$Z|HF3Ghs33bdV9Xo~VkeTZ0|>4coU z4pT30E9|cD>ZEgH)wy`Z^xAaZ3f$~Z=)=(3|Cdn+^#8;LuZD?nHR*rfa*>9JO&iYK z1HL~~u~yN8Y&44s&=>Y&=l@p=Fc@e7g3Z2slx-_WO9*YvX_=tx%>FR^I-L4}x$2+F zYqo+ifcCKhAo#{#Bmf!XR`|HnOnafZOzRttBgh)te%MS*^s6WJRxD6QLmRttIbkB$ z9%CJMmFs{i_7j6)Ehf|0r*w)eg*N_MulQVxsnqL-x-)+X`?VaW-)TzmNo#(h_X^2wD zY8|%`dWK~(#x!6O#=^8`D`zE;-d<*og`!J-pFS!OO8Ru0^zjeiur|0B2pkH9S`y!+ zX0pUlo{tff&Hl(v{%lSMe!LR%=2Xp)l5Eqml$3RDF2VT5{Qem@l4w`-qy200=I^w( z#XZ-x^D#LoGLFO9`VQbo16GWEU+boVlJP?+d4<2N!Z7qr;BW=?vT#xv0Q~NklHe-q zxip1<4OHHU=R+UFpBUVTCD;X}4tEu_+Rn658)dHay*Q2@xz~A*f0n*~ zChX5x3(ckKY_I1`pTYwx>hv>02#sWLmRt+@uEpI5q<0&H;bYtwa_K9nv!;*OzP>N6 z2QRrzGfeng;ToP<{K_ilFeUuXYd9yao4;<;JoV!J0B~u`eDQUiK6Tb}dNDf;e8`!@ zgV$|nJAALdfScbKgiCctZ)5yD)v5ZjZQ=0rF#qy$1Z{|drIbhF>m=wq_S);a%ksBg zn11BMjmNICYBgZg9?x4y)-@3>8*5`_yn)C4kQ8?PJwO z+hV{;adYuc^l&QrZ}c$Ze?kxObvvcjE9I-Qfi#H-#h+N4RXy!}cqH50K<8wwW#oN> z<`N?o-@;7=IVOLXtK7(}H`|tU7iS)ddD+>oI5cdyycr*UL zK@VmALJtLRfau}S-{|2u-oK)UTOqCkQ1ZD-j8aY`>DPpBE_973M174luj-v7XhM== z0z;j)oRmdEYSumu0i8K~W*ihVmC&qEz^Ms8S;}V7i7*&2{TJ~E>VNJf!EG@l7Oc%QyeTxlb%dB9cqmo~H1fbV>>9GTPK`NsrvGMIyo~{xst> z`a>AFuWUl;yBQ8Oj0JEilcpE+olm3->2a4FcqLXT@E*=_hYUg+mZg8HXsp>EX zD0o)$*6FKw;2f5&m>7_-xx=s%MWn70Yj zLPp+|*E{Cr&c633R8A>m+KiFi^^jRDkhvi(`hS;i?Dz>tv)J98>>jc<^Dsq8zp~pHAHTJ8epKN@!+lgb;#752uMWpqO>AVD ztE=2@a?g^f76MO^ujCq^`Z<@&F_#K=rHbcou`zBuj_S|I6Av`+rn@(5f>sD`M$TafTjvyM-z`j#WL*c{YdZ< zK3!Uwi`jZW1T-SwPM?V)C?e?s2`?T!4iAMp`rXzDVy9kytMd$HUgx+1k8$yKgs1Q7 zcxv!_yHbfYJM^W3GL*P+HTZC~wOizq+V{=Qe4Hc!c;bnLzg_lYh3wgie}wE^_U>Jh zfAb%=9e$g{XCXQF#6l?=HTHHb!`F}K-R`-y;XQpthg2X1I9*+;1ni9@-MOWX_-4kb z>yTUJ5xi=n1cSNH67a>7sq5|PQ>|QlI=-{JUF}qUCLzbslUKFhg!ESdt0lM+Ul}5 zX!F|kmY&XY)5O?1FAk6fMw_jSqdPjKm&Ov$am$tRuBpbPL8rVzeoE*czpAYe^k;23 zzrZ%#@+@A{8>Pf8|In^3_LxGGx_uUsYU37yd;Kg&0Gm9RbokiB1F!AhaMhR2m&~y0 z`N5WeHS%7U>t0ZSe;Ks2yI`RTyYHqF*nlA}Ufm$2sR!XRV9_oax{Xw7dI2(h4wqV@nOB*flV;QV zLd(`A#(JA9i*S1(Qv9czfJ=UcbVvwO$?gE4ivqHvF_Db2ycs z)Ln3R$hMBAU#i2J$v7)Ye*b>IS&zaDlceU!NYrw_xo+cYw(Ztgg*+?^m&oM2##Os|1o=BQA503;)k#7Z`^iWIJs0zj`dHJg?EZY368D>m(VO132#e7rP5DV8@X?`MuW!>w#@K#LLpOVwPE71HUS`N{W+l$7p-N@SKXKwzRRl zB3CV)MCX)M>-iMu5d4DER`sw*q#?A zikryDm8(mjlVIHVmEg89G7n%b@{FXwOKVYO#ENsR;Qf5O&Wrw5?xVLr>rn|f7?2OE z`RGO;SE5!w?y`(oWl{A0yZlhk7?^#HWGo>ST(uM zTn&yE+eer3JI%{J!|1O@U_hg9-36XS<;M39XrO1kmXDVK^J^+JaJX|ScmYx>xcJxV z0sJjP9!HQ73VlQnN56pNwoLGnCXrejl;$#4|0^g1UipRRZs!sqCOUBfCb~Em-AD}q zsy=+ROiXxvP2}+2hMa^su&j6fkv_Pf3SeU(q$^aGazQMS6u7;sL z=v8($>>vM&v2%`&EZnwz#p&4U*fu-1ZQEwYNyoNr+fKTZbjP-B+j-UJ-1FYK?>^kW zs>a?mMpf-^SB?2wbFTGyFl?v2(RoQ9aXcJPHS?PW2y(gh<Pcy-oX3GGE(=K(ehgHQ-5Df@*r>LiyC31#%x-v`l zMmaX8c8(X`>1gR(eQNTaM8OL0k57v=JL3LSK6k>Eo1(~jZm>=)=PK10>dNIj20fAB zagykdn+5J@0M>t1n{=%_@P~3#+V{gf2|I^wB#4iJA72t9L|7=I>Q#|7nuT-t;SQ5VjDj1Ne%*->WA;>Jt@Dr6-mjyaan7K2oO(c#GHGbHEIW#J^ag0yt z!Il(DYVSgvG1+9pFQAij_1w+?>$`bM85m|0M*CdcDc+bm_ULks#kMT-OoGU<@4UDt z+FP7=2pw=bGha+yYq88<&T$i2pCH>KL7nS>wHXB_5;Tei=w_{gCqpbz*2A zwMWT#!xgz2L71Ajtk8WUke{(OVX(zdS^0T1e72NG5#3v=aT`Rv@usz4M?~+la?`?g z2Y+C$wo$f(h2XY?WpKBI3$}B9LSevFF-Jg~$SFF}BTy21sNjiSDUcg{=|h2HbcUpj z{|1m&0Z6;#jttKnr~W~iS%fwK?E?p8te>kBGcc;~n#*Zh!u{at$C!tATC>yB%PC-{ z6h_UY3;L>cy{u3T_(R1kf!BZra2-$(&vQSzNsv>_Yxfrb`~Wv=N^Lwh--_+_Ch+HT zi{;D7f|@_C^5-jZRuKRm0r=3eRUyAc{!<^F~^ci?SYrMuAP!aRpmVM;5g~fV2!$U|7P)mC}O*4Q7k9 zZ@61e%nLGYZS)aDp5d}pU$xtz%JV**o1RT9csnFnb>c6Yf=aXu8FieCI_WB5nQ+;bL!9RcKtr`} zdz2QK>|^qJ5d#dX;12IGL{x*C7Ipf8B!N(ZDM04QFeR0f8G-}S2H?*a{6SK__%jLi zcjq>NMM>fTjHv;}Ib+PzU#t9$iZanbBS6r=C?OSOj?d~)XxKc44LjH9jnS)=GtZa+ zJe8n1UkTfw{UEPU+175yD$NCQfy)5!c0=^e$qHY%tO0lzef^~d{{&F`q6JU`sk$Lu z0CU}0VWC)4`JN2dAoVLD!pV(*UUGv?nUNHY_C;wZcpepi^xCU-1qWSUY7Z_TGAJlw zhq5F^Db(?UEe~1%*`GKCX8~F?sWy}$0JB;90T5y5HizcZcLL!btI^LJyTA|TdA9!w z*xH6gm3MUwMGlAdL>4AMy;o#|CVm_!un?m4>#8$-qfUmTZ?7asrP)e>?W6m`)JMnU z*hd#0!uE@9)I}FE9ZuKiXu}G8Gx-NOn*dQLfMw(Dw37g7kwjw=VBrX@_-5`O$?HP` zV4y}oyjf&Z`w5lmkBDOAk7%?5e1x|SLR8y&js^)eSb!7h1>oiq~Fjk~IS!u0GKES^J&tMJ|iLjD8*eur|`1M>EP zLgD$V?D@ZCf-v#cp}@@K2$zZ#9PUM1b`}94S0us5+rxDmG|kl5rOKb;XUT72ny=g! z@8V~RTO<$Cz5}Pi?_q&B-UEWevF^&38Y8P5ivq{+*NMUss@`JYrt+K60}58S!d&*X zNW$?^-gO@^r?on`I~2gH7?F!s+bb2HkyAzWlH=(l| zhhe+vv=57E^CJaxOT244he4eW%=_=!9o2gs5fOO{6vzE}h8Ze+^xgxhY&``K)OmhQ z1l-j_-fg$FNbvw?OX%yRuta_p2i;-ov1fV^@YD1mXQ~{L^(}EZ!|aNHVvo<`^|tHgu^L!S|qP z_v(fVen5|#_0}6TB%EL7EfDEp?$gYRbJ*wifS>*57n!$8-s@Cb=kOfX+f>;eg9AjC zPX%1F58$`Yc$Pog9&kIyzQ2bsdJg5Q{3M!#NL%P{^1UprW31Ny22*;!ty5_P6(^*t>WyGWgE#*L+ktwAM{Kh+;LOl-D>VADXRb z$LLf{cwoxw8MGcCgHycJ%}H?f$gH_7Ydwr-1f)^C4mzm2tMvbuch!Y_nYw`*#+e&mH&w zOn1rLW}x=t8x~5^64Y8+)LTp*hGF3gMv5+emhR_{wKb$+`C6laq$|7*m^R`EuWJeF z|8T=_x%Zwo;o#j1yuKbwuXW|?;uPHhY1mm|8k{U}XHfBGIviofjiAuc{lMdKE`fl^ z`ChDk{I~p#Md5e(ALVZa9GJyX&&D^mUjt4DQTf3TN2Yk>YIEv?wr47k)|iUXFd7@* zXc>UB?^x(}oEfr@m8}-o4vtjo&qBp0;8)PBA~%!9=<#&Bjq`K<(Go8m>V#GVSERwg z6%GHlC7%1>)BJKh^J{DMhvVnD`j5KI-*U9^No1QPtL6<4D_CV?!rU2`B|qd!6T2ynNCHXe%XZJ$q#rRnK{>+z1Y zSF?2|-@JP0>SJF9e|Bhv?~8KBW6J97$fhGv{UC6l?BMbEA6 zwtA6qJw|LXv#`~JuF&d(=u`XlHN{kuV!h_6)>O0cv$>OELDGEkA}8yiD#vH-^jtGL z$7f{~Y{8gS0L^LWp(?qTO}`z_!)TDTcm0zoyfQ)uY1R2^+#As^v5UYNKM1pAcuUtS z=$R8ogvbbi3>SIBt;}97-hk%IxokT2{Vj?90XS9UCcbQV(q5!3I zMh43=2uS8G%sYs5jguhcha9k;?d)e{7QYk=q+ z^~=qsQYrr-mNRt{ZbwLN8B)S5=n6?-Cje{&Bg!HK|8NtL%6SSMhR2hituAznP~?No zD0L2zP`K~u^mK{6sXE0}!m5e#Gxhet({q!ZBk@BvSL0nt`u z5+;6WP`Z%EFE0um2>%LAW+IXo}kFk3$+63}3r4`rHO_c%ixDMbF8L zL-7VlYYTIjl668qeYg^mQ7?DuQW{bsm39i7Ws@S&iX{;v>{OSh9Pl^%xd;$4F3h=~A6 zYvZ^_QHpS<7An&say$#00*7V;79tf*QW9pkM#T!x50%%Y6VL6l$*QVvv~Y5VPj}Eu z*|#K9%`yJ-eB2Cd-~609b=~1G$3tYaakx6#hkpA0${)hjQqg6NLXHxPBvtrmMC3P@ zwbC{v8T~_wN{uSn%2l~EzP7sKmzb`q_=0D`5ysYWAUXUY@v8}HQWgTAUQj7uB z=Z`7??T`2qh@&Ccyrr7lTEfsvmQt?cz6hxKP(T8Tt+vaxGbPc`wuIEk;0!@x^d-uA z+Dsgou~At3;YXnu3~9)^^T<3A=B>fdzSnVHmLOF4Icix=2Nqz@7pEnef4?0xuG3~t za=-DY(mP&1pM_Xbq^|C_eE`ZQh(b+o9^)MK1FnE<<#ZA!a}#@5?ug_W^k&G>xdVI;{BCQdY#94O9rj8&2K7Vn zYP7_alwc9fXlidB=jAG~(g-#!scY-C=6&qZgMI%H@37Ga%*I^5ydd+)gPi+jm4y5X z?}1vFXOratTt+baY;+kJAZZ>B}*ni)6$X z(7?eX6H9=9qDBE4xrDyr%d)vdP}|x`I#kN8C#<=E;7|7t21g}U?Kr(IBg!H@GzPo{ zzeVpMA}akwH1I>6sme9(L)|HY<$0Zr-BVurjGa$E!59Eu0EeS^pdZ;fy{^zf%^E19 z6nIcNomMqV`#g%%s1_a009UVG;Lj0WZuMqO;=){(%howTNoS(SWlMTeWDh|diR0K) zQehL!d<$lgM4}A1^yE2{Ayz2akR<*LxOiWML?RY!%fF`>0^zpd5~lv>G!ijH!at`y zcnvcvp%Q13iVXPcSBN&8_|oR63oZzW=)-LJSSll@^#)q5-Iq%Y%f>@>OVBW3*tW2vyPV zS#&(T-2?Y^x9#>??C1dXdtD8_)DKFg+4;2~u%E4*rlRx7M#6qS!d`>*z_T^eNnImG z`ZCbfk?aiM5|1il^OnO*K~JBk!&NDNh8gR`j~UQ7=d7bmPFa7NvrRV13j*WNh!pBV+raTxG7v+4Wp4 zN)cc53&dBI-hk~{91XGucN~pj1s;C*EDryzh-t#9+d3WK%jyyeJ$*3j9$$9J- zUh}aX19_pE!1`yt&0XT_%7?S34IU5KSKDm9kV^-ie&I3AdTebQo<0$}6WS05N}rP2 z`EWLMj-$95GpYP^{IWk>f@|W1! zTp`q&qcJNTnD*Ah#eZ#=u9)q+ltyWE`br`eQy;nU0@7RCOLp@q9Hl0Ge3dv$3h(`9 zvTI;T(94#!Q1v}{R*=>?{^nN5(tnis^7o$z`*xbSjRZ%!Ox(iZ9WBT?TdWzU+0!Ko zM6Yx^*u^!uj^9VArp|RSzFezlt4QxRvI<<}xzrQJgUDymkUicp>E-CT{}J6;X|{wovt`Gox^rEv|rehcwKuMRMTn-cs$7Gu^2Eb2R&I5RMTn+muNxR z%6|K^5YKBcVph*)=+xCLiY-O;rIeZ1+Bo;!9QmEp2ZDl^8cgr~d`UwxSNNXFfnJ zV53+AH)bZET>G7^ZF~^t#~P0+-B6|!wf{!B1Kv}1vPyj;j6oZr_Lm4}E;5HKUR56z z#FWT_W*<%X99Sb5!f&ZHV6)4G4yFfoPl$(uwh;}tL9$bBITmbl+2(~oJx+iZItn1O@}@NPKJ+Cq*Nvq&zjtuPa$(hsKXqh+ z!G?m-+x&G}Qc#D;=G9SLaRnb}ogjce)u#D1!3Ro>vGOl4)IXhb1mH#F2UYuA=&M01 zN`Fx=27{r;W%E{UE>~qXUa4r@vnlf96r@JjaER2RYhf2IUh-Q+(Gb=+aM*Mu1L`a%@1*XoOZ04OFIVQyU2tost8epKoBh+> zNld4a#FcwFO~~A%(yLJyXMy-2W??7Hq;r)A{`jp$j`%H4#vv8*s!@))S55RtWc6{* zoyUdCB(wSt>xohs8D~C^i;UaptjPxsbL9!~4_o+Ua}-|Kp_;wSdK5DTqCEfKXOS~F zjin7Zjn-o|Or0Vx>v zu?ub9Yj9o@78qxyv>jnqseM@{3?|OdW`ifNp06+meihlCHSWZ|@s)?eF3U2>BZx;X z{H6qlzP&b=fjh?Jx&cFUk`aT&*tl{SDpso+&^3v0r5VDF;fq9m%26b^W=epB9x4vb z&Ox>RMg1jAXFDnZ-eDKU37Su3nt7SWWDbQB65eRNremR$bo&z7=7%k#(2t+^5 zkvfNdRGt5F;K_??H(ZV7iBZm*#`kambh8sXY!P9~kT3Lu3Fw9OnaqTLl3tG{Fc|N+ z$4B-&p4j14F<@?IkMWP??3Y_YBqZqVVIRBOECE~B##M5&m5CA1N@N~TQ}}4${%w`a zuj`zO4T&HgKJvy=>3vK&Qe|Jr9<4lYo{cb71SS~y0^c|XgWi#0aQ7mg= zT^Gl3<+}yx?`KC%RM>J8T^0`Mi7>VdRikF>;720yo|z`4q|GWWr)ebAtQrq9@fm6k zY2f?H1tcM)W=J~x)>G9{Yc+FDPJNAsTlk&nfv0v136-1nxShl4y0}aJP-%1HxBXBl zB-*;-dp|d(WWx>}bmQ+;J_^U=!R-+!0iovu*RL06Abk9Vaj9VHSkM+7fSOjq@r)Xoy# z)ig6^WBpV7=_murv6iBY9?jZ1*iS-vXme# zed&HFvAJ?_Ch_vzwI6lPYg+noq z(iklTcOGHHPCtG?gBe#*9H%ngY*=+$9J5US3;8tp;`ysRp*cZOCt~bevW*wYk?FT#+BqC;=Y>S!?iAHJxw@~%B)=1GfN=j z&hCF$I(4BrVQKH*fj1iS>11o|uu`9;hY0i_^WkL)PZ-ciDg1NflAV?qd60rXf(lZD^wCGQ#A+m|LQ zGE|&<>=5qX!wh`wW8x26vzSbHSh<9U?uf|EUW{rE=(fcyV)C6EKdZlJB@VRpL=5u5 znD@Cb;OKNg9;1P*q%!)0of*a`F2x-FP$_#Snb>c9HYF+@OGYW${3Q$shee>&PD^Z9 zZ+(6g9>H}9294d2KdcmXKqCIcsFyy0iX$?3yLYuATu~ckCS}t~`WyvfzR$+DROv;z zH2d4`Up`n{tm_bM&5!nsjtqTdWWqe2P=P)Kj#wl$kv@#3$%AsH=i)2v-f!8Y?Go6G zGEYcY`k~de!eF?2+)gw2_J|~ts8}#v-)I@H2a(PE8JM<+5IDIo{ER+@S#ftJ`z)}B z7m0|HVEJSI7w9LCF9=CpK^W{1{}SXFRuvckg8oB!$iIVrM%;e}{p0@v`qg#={{(%r zUk&)OUa0>B{lREWkAkA|U6aP_y1Z0Eks2 zQ}yR0Q^)r){<+mY!CAoW2`a|y-(N}je5r;|hG^1S3 z#J11c{P>0{>mvEZd@j-bkpjva$oW}`lofKr)r|9xhe@IbGj4S8)_a-7ZuhbqeJxzmJQ>`xPV1s{ZgBh19#A${l!MAb3`1*QCWiiPRqoX+wC) zBvnCpp|QOJlg7sYT7)L$#ynbx3;|>Py$w%^qt2b)=V;K2RcF5qu@5wk- z_-h)Lc66KGlN6riK?r_2Oi`{+y9EOW{RGx&t%_;tFUG=I6xz*}S*4CSzdhQ6etT@C zZ+E>>qr=~^PK_4juuReDY!=k#{PsM)HToI-+jG#|wdzoT;f7`co-%9gNyDdkdhSyn zx~5#aF_xl+%C8~|{|0pLS0>d!CV8-XW4@3MJJrB`2#n)=5t8~>>VZ|@(awCKx2Yl| zew#8Re&E?y;Ni^-v>p3XdXbB>Ql&5*_u%_IW|w3*9Ym__bm2ek_sz8TBQEnZ6I==C z->G!Tgcs&pNA19_0S3??0#PGGKD=j$cw#g)nMf>en@BFA^rp z$ZGVIx_buvTv2M@$r^qoH`g!Cbx*Y!E#3M{Us4nKW*wp40{1`;2$I~zWh$6?vTlE5 z6%8^z>nLk@L;hK?-|xr+zQ9Vh7tyk%sNP(eL9uw!DtND~IZ>+&A-RVr#ef1PiXJt+ zdw^b-aX#!k^{veVuS&#uQXHzcR}?NLjy*uT)IV9`gNr5)__Q!ES!CjN`ey^+yVkpj zQEaJiGTO#$nSPz%ePWQ+6b>b-7Y}^Zo_%gEBt7c3a^?2Hd|kp%KVPdhlQEp?@XW=5 z5!);v{C$?z>EH3pSM$wmO067Zlx8>nyt%W*%k|Axl(1(q=y1>gM`O83smFR2Cf|Bco9}=8 zOsV2%cI#0&uUgYyqMu!+H0V}SvP3*9Nl?4su}S|jul1tT+~lNgME56Q znbZ77qFmau?y-Wt+p+aB-MofS~qlP-M;}H08sgtNu!;>pW z2;TEbSb`I>7cweeU}!eGSY#EygUQpEJE6+cSGkc)M!BDl^R&C1!}BT+cVv1xNj87= za=Kp0^eRb^;&_r7Ot!m8N3QLjrbtz5Tc${@?%AgBs%Yvs_??U09s%Yq+TIZ8zO3B5 zuj#|cfT`S#O)0P7m1Ij@?w~xS(suDXPg62EM2%BQBd)qjk}_4fLy^*}qNDUf!a%X9 z<*T(FF7*)A8c&k+;<=%@d~-Uar}M#Zp7_RxECcxnDtmh#rF>ILWW>!{QcDMrvrH0r zZpUv6{d1K9Tlcp2OCvtO*b!9YE`c&6{c#NELCcCmnxMPhQ0boPK8}rk1drK z+5y7+X~+$@)gprIplS`lNU>@SQ3lMSu6UC4g06HDFFGB9i}OLk$hf9m|KdQe)kmR> zN#%2_R=(<&I2Kud3s4bM!VG|9FdK*x(aQ`p(m7}ucASu128d=dh+@~LP$HqJQ3m)l z>5$4xxnd`|U``UaCXj9n{V;396FgyUS#hW=xI0w%Z;1-~1qIyh~CVo)VzRY?-1I$TXRpix+6aKa(T(Uzsn zj+D}qrCqWfIhQK@MNm#s1s4wouIyS3Tc6DKHma1mGsU};IM9+mYaIbDC0Q;M0*pnv zWsaZ(5^%(5AW9#ax@lC44_;MJ3=U%|3@UgMO8c zxaAlctP@1IBNpg=>>uCmGGWU%eopCbLkIZiboN<#LkbRZSf>j^Q_pbwzzB-^8vj8` z`9?(uwY^;Eq>}fc_eSlYm!v%@`J+N{8LsQF{Lb+Q#W#9|<<$8i0YMvUIoKm4d>AQ3B~pO6K6yoH05r*Iz(_F=}`D8qBTVKETe)dgOZj*0Z?X zo`0C(j|BCtq(c?2BB~YOS0LP4ofxWVDI=CwmEg19=aoZ+-z6#{e%lK#e_GN*jZihtp-o=nI5 ztxx4Ku|zYao`A*G9#m3mR?Oy;JYw}CHLIS{lOSOqeHazvR;EciZdBbL2&qFREUB!? z5|A;Hz?(5rfxMui7Ja7-q-<*G58H8(F;Y(inV<}cacY$TNaAuUY9VFhooN;DDV3lS zV$#V2UQ1TY{C4PZ(W>D>u~}p#L0R!3d|B~1XzOnU|Kx1E`YX3q9_tNevE>&$>j$!G zWci_M9<0*3>K%?oc3sBX+)pLGe|Ij=H_vRxf^Mau2;hlv>f&zh}yFHdmvTx6G}++?5F!`y;fw=BbP%iMs>;|*|a zWB_wBkTg%s;t zNnh&{RAUTB$CB3Ev0L8m>&a46#L|!de{ws^0$N1P}Uo5XD&gRTF#(H)0RyxnrpYs~J4p$ca-ZKUZK@^`C7lm7G91!W%iWlwTh)F z|5!L6lya~ATZ@xx*9i8Kg2m%??f8ET8rF~XqH`8_jFEO#Sw`o`of_MlNQy}X7V?Dv zboaFG-qpfCaLrGkg72>f4?55hTm`EZxc8FPupgIje zn>?O;3C3cy0mB_I+XXy>*k`<5A$f!JiYFCXPB)T;JW;>}G`S2nX)P35+6N35CHuPf z9*GhE%Y%11PnwwP4X!uxvm_hLrcZtGkqHP>ci+8#g?q<0WWS`;Nunwf3kJ*nHG|ro zHY&e>O%l*H0Ly&6_L0V2go#vSKuk}y9p#yTDma|q*Sm69%WZ zj0d7((oM}|?#ae{9^JFksp)oWD+Ay!IYi_*CE6u*%q=(<%lY6yxqyJlgnRy6E?lkg z$0Vm|QS1VN`!o*2{N>zeg7jv0jp0|BrCaD$gwVN+vm2B4pS)e|5yV$nv>iDSaH~Nla8r zq%B(ASg3vq+Q~aotJe-R#mWwx4qOO(%^B@s+Wb$Xj*OU3EWL0_Soht!sf@t4kA;z} zDxS9rZpK)+YA?o%pJPI7C%-rt65pgQ$>=;q{R27baZJI8IG!@VGSTO7)6P6h5YEQ! ze&4JcA;`2+PF)}~gnM(%??sa9OEpaGNtf?6lIr)hG%^UxGN8np3--1}K{DWXUNXNH za4RG-NXIjXE*8>=fsJ=tTok`euSpAf!lr?ack{44HULG~TkMRb05_o}LKWd{8SYa0 zK_4rWT0-zapA=bZIM^(Y%zP~%Xw$w$<($Hhh zA36-eS0YPdpD@X6skUF+@K47CFK&@Q-G@Dd@6$QRl8@jria~JRb8VD&+8$Da(~)A5 zpKo1W6jIYjJPF&ek?%f4@;J?O_KmOVXBA{4=$PE&AX`c{@|=d+g9UA6{h71Pa*2l9 z_PL#HXz7B!eWN@`Q`nvL$1#}!M$;i>A$@p;X*S3z|nzaSeQ%Um^a_BPYL@Z%Ki^YJN@n+_8mXj z+p&$L1QbBi-eyoMlGoQ43n2+yy~KX&ghpb+J%YK`DH68v?d@&)Te1wSawIzXX#N3q zDf>I1q2wg;VBsW^NHSS(s!z;CZ06s5OY)k4!6alJ<^2R}WT+(Hsr?qeNm1;yJ$4Nd z*gsvso34A%ZbS&B24~R9>EdSOBU}L#W63Li>BcOUwt@RZyXY zK=#{RPN&A^MkdNJAVYdel+PrZW{BuXK{VeaF4-wq*&4RK~OnK?aI>Iw&s> zfCt=GvA>Sj$N~C`P3V#+f{86=$Sz3m82qa5o?{kq@WW6c*ikd`4=}`b1Lk^se87zm zIC-Mn8%4N&iZ?bY@YcWURu^ug9ODk5RF!2<_bm^>Q;d7v3MWPF$($imbIHw>o5epVI$9x-Q89Pb=MyVsev_@HRqum_&HvAc<8N$4L|DiLQR0L#RXhE^ zMo7pYCGJ%bJ*fR`Fe2m6QbhZo+yj5+AjsQv>TG`7_N+Awj1GVF3)3`@dc9~~7{+n# zy01%qIY=^OW}bB=hZ%3O9zV`n{_Am|NvmNnm!b!k{S7kS{O2Gew_B_A`M-;**>gN? zuRhNx$S@&<<(%f5)&rVW5_z7WG5be95mGT*ps3OT?MXl0wogd?zl$oRHqk&%{*kdv z3xBQuzQa+MT)B;oV-ygKjIq+{Ckm2}%V{hj25m3{!*i5Z+b0jMa6k;nIqth>j2l4o zfP`g);nauZ@-xm8zq{WAoJYov{-Lpgd_Jo-Q`m=}IBhf2c<-6c91a^%b^?jnCoDX@ zC#}*$+Quu*_-O6S&pKd%v|j%oS;kx9!Njc{Hi>8|3;kaJgt|~JF~ir(X+ZHmm6hgB za8^S#!M`ahPxI4_e=|AgUWquN_O1h4~68E$OmUEI+?{6%b`&F+?qEW!87Tf z1Lyz`0x2@1B6{2;|fC3;}Kd9ck&vhhDbcCI ztQM^SZ8!mU$0d3utYkL~dFWyu6^6-R36e8xd=TUYY=iZAo9GGDH|d zdW3HDaOnO+VA7Y1YNpOHj&c^*xmRl)fc zjpiFfz4O+f!JQp4qy+)>xiHzNfQ4SNevn%}O2SI8RHwqYbdLaL#i0Ye^@EiHMcB*q zoM7Prl1j?Mee*T-tJmTsl&Z7*aAL2^pB$j(XcCt%&wDtE=Upij_qyZxlS0QaIir6! zkH-y*9Lpv$hg44r3y&;K#wL=5zAPDbqfbr=yw)h2u^&fz#0I&!N!X@J2_5srN^`w`gXhK4eh- z0olAe(`XC`En2%f^7AL(l^D;)?`jHQ63e$kfQ0sMHt(K43hflHKsmlAei695v|7`P z^H5pSYsOh@!?RecaqI*dtPWo3j0)O;(8{=}({GqgNIjw@BSNIYC(To~`)o5!-UQgB z`LQ02pgoH52aXHZRx*?rID#?uil>XReM;kLd3=IzF%6k~(ya91(Np5NcnnTV*!R3I zpj1gNPu0nsnjUwmj|B4KMHhWfndv@ObAk4H8Yh=dccBL%BA8rc#A`l zaes6Z{A3XBS-3Q_KAOz#@3rVHz3xkn7HLbgfe; z9*r;EFm+U~qEz@tGu}K9|5AZxCBD+IC?sCGU!s^19!E4K!V)e0utw2DP@`}t+iJOl z(g`SB&LM!pO>8}5o!{xAeS>1R&_x9x z?0kapy}?~D63m7Lg5`<@Me6Z^_A};~A^!XU_z*tJzv9~)y6o2u>)r$SV$aFX_};&) zSirp@ik6R&jbiNZPnku!QKp4kJGob*sEdN zC^a!jrcIRANPnJSRRy}U#Bjs!Ctjg5J&l}>h44l5Ym|yLPprR%HIW2})e`@)j2 zf8EJhpY zmlutXl?PLMubI~Ws3|$Zi;2H1xS1cm|5Y?Rg=l|vtjJO@&qefDJ;J$lpRLs&9#mc) zmNIHx%M0EFA;xWZcxY+0bV!9Db@#1PV#|5b3?X3;-O3~TR_BHQsNOGE7+u2NGP(u< zPk)RmR&jeZcJ@s^xAT+fT^@3t`w1O?Gi?<7Gqgd~Bb1-c-pi#+({k3b&Ss2?-=Z^g z!DzWDyjl72pAsAA6PBgs|C^w3`#)5P7F=#u-wBWf;hgpYmNo+IBY59rZ|wfBfelfA zVBU9G(QHEmY%b%&;8!CE4}Y-Rq?4Q%*xb2>+gqx&cHnHx{+;I#-zl7}#Y@~45sU|L z7PAg)?GzF!?OD-O&pQna^nM?L zBo8)Z?)C_eoMg+;$+Q+LLP{p<|>m^y{p>evaxoF%YS6d`GfYWmyc zPS9p4_evm9^a1Rc0;4JDNZl?1B8bzFGCtN1b)$Cp4tI$h?^6jzg#a06z`VuBd2qLRnNl9Y}9CV5QzzNu+D8BIP}^SDz5ySjNoFXzCF()`EaORSi~ROy_pg#J^z-iF*^TE90J1 zO@J?Gm%h?3?0eD+;_%OT9Xk}vbR*GHfGa{8R@4R^{dM!Bu~u6{C0?87T+ zg7nLKIVY<{Yj3)E=eup~>ZUX2AVt<2g3T1D;T~r8v5pHgf>BzA!7erqF89^wEBE{A|4q$sTD zfS%A09{MI>l3%4{ZUL0f?g#gaRRfYgXhz(h!0CO^1L@^e4rUI);~x%{9}hddKfidx z7>{a@bqiwlz<7cAyKT?Xdu#7ol#o)TB(izJ`@3y;t+3N^{054vcUE|Zb2U5*?u9nB zT`dm`oW&F=D;;i{gq@WRat|j18pb^Fd~1$Qd$BKcyyxX~t5A-EfG*K0occXY9^a_6 zu6M7=ns+anCO_S<)ce-n_qT6-dc{!m1eH+Wg7a~a)ZF&p`W0m>er@dyy06gR96@tW zfgzY$>5so2o|vL2e11K>eQbWI;wn>O8W-GexAzZ(`BNWb1EoB3l(yS~qR=6zzm-1? zREKxjY7F5ZYt)k|HD&-dFv>|G7XRV|hB%O_j$z+cECGV&q;-nC&X=9iY*XQl*h}Fb z&70vJb`*GNI^>WMKE%}WDQ78r5e>_5fnm`KJPdTHyWbzgJU-y$)xC54sN_*e>b3@Y5eL*iQy$o2>V`T?6?d7WWg1$Zpp5NL{9^?R;pi!I;;`Mtq5XV#4z;*XcP`?c zZ_mbLs)XIFukT+S8v`Q@)Ujx(r#>Lx;hbZ_${U#5<&Qo2@#6|n0j?52Eib9*!PN`a{|9C77#-Q7Zh^+O z)v-IaZFlUBZQJbF>DabyJL%YVI<{W*-shfs&b{Z2ci)dnYE-gDR%NA9bAA(_zf-*> z=v1lYTBY(SD;9l{Q?n!rZ<&1Ua@Q*%2vylLI>n?aPIT)NWr>VIw`fNyyOeldF)aDu zK?Y&}!hvWSY28kCmu;d{K~O%jFv;LU+~Ia_?5E80bevkPD3vF5o7PJ%i(j$bz|?r> z&m_sk99ySARUTO!CyTqqnSz)lYJ(ia-?$CoI5?X2vyCtFDOJMAO6_|sW`j)y=LmNh zBY}ADNVpSGsk`gl%GYDqab|wdMvs%{&7ncC(l*1rg1t~IoOotNMD2U)nzATN`(kpt z$#`Nu`H5~Ubjcr++9yk6h(?~J=3l=SNl`CV$RFYu6%XAT#zc`^-5bKiK=dO`AVQ!| zU-9g`K=i*UECvc{80ZB0*CCTW#SGnQXEI0-7C6^YN}gfiOC9?2R{A^H2% z9#fYJAlpt*xi)#X4hVTN76Z20){Rdba$jrZ)e;$d-5xtswk2{k+eFr49_SXy_vSO^ zN~7RPXL=tVc)9@Uj-Fvc*Gu zzkZpbgS_0a0KMEb@bPw1w#pV4r&vs4`aYzPLUd%~TrJ4^{9!RyC2tk@PZ${&r|!I8 z*t>;TbRbR2{e26|fxUHY#T2ra5_n_0gU@WTmey8bwVfSLbDm?MSim254X0eOELFkh zXVp>7DoSgML=+7tHR3h~(|*D)=_v(HmhRW--R?Pw=3E~}<_x9RiSJF~=tr?z7G$>p zvI&y_1*5S&1LvjDmf=!5t^c1?i0p4Fgf<$?e89+oN1GP4GAJ`XoNeC=ocHvPneAY^ z6C7fw13mn64V7LqjF5l+v}A+A-y>Tbox!$G-GPtS0qCCYg#B61B~oKbMAHK)<@E_ ze4TIdSRy~#YHWZ0o`<{-{nE1IXHW=&$;FC|%rqqSO0VreSAT8Gix^G8i1q8QCp8 z4NEV;b>?|EG;bd;hLwyilYbz<*91Zat@L8wlH=qo-a(>f!EZlPHFzdFuHxp{4A<>-Yp>yCxX=_Y2|s9< znhF031w;Hc;&^X)aOxTylB%a48T@eZ{hI(KkXDSP`Vh=y71n2oYdFERtRShPZba3@jJn3v6@ojw`bc8>=nx ziU9G-x62Shf%ytFFm7J^>;SA>^y@z+?sLzJ&q`$ulU=7FjaP?-o>dN6x5MDXk! zXg&r$Ld`e^vi)b7;;RLPDCH64zWSDc17lV~+KaT~U0Kr|@OCcYeqcqce@ zLut^fFt$)4c^|3RGEt1>CzRKp`7C*8#9#Eq|9{gLPLhAnm!0H$y;aYD(wDXWqAyVf z2%of(=#SE0sWzRc7B3A^M7$9A(MR)qvU9gcCJr^O@?-?jsj7{C&&2JucoS`_hY5Ai4?A5$hBpNe^Hn~?ulWR-R+)YIxa9s zrIareVe`ZSgH+!?A6I*-Lc8=lAQ?YV9Kbq%^o*kvXZ{y&LH>)kgu!~h;N#r1%pgQT zWhXB~=k4#U+}ln{WuUdNaop{_AgjQ&@qGJ95Qi$V8ZNOBeth#z5zKwkaHbHR>hfjG zam(*)<^fztF@D|RlW3!8G#|0DBeCUb7-z{s`|GsO-m0Qxl%=Vt%RxhUj~{+;)yBC< zIEtnw!GP9bw9UV>6oKo1vXsw!(d(?!&)YK~&m{a3*v!r*d?YXxVY;&O#5Fi^R`-4N zqj9G=aq(q;d^<{|SBp)4HTndL>;my!e+Anlk>{k zbbF&A5WpNpDq~i+y!Anw-#1biKDyVQKKUX(UEfs!9HZ+Ab%-B-%vNxDp@z(W!{(^y zJOiZwL-?%M6uPx2yf|ULbt!_k*;C|ezglEJ~ zNG_6+M*-t$l1u-tmB|tAAAK)$@f5Zz4xThe-&}pz=fj2QOyXhKXgrmh9I}r1j>CvV znk<%X|Fi{Yxe&})WLqm4r^eTB%dDEqPAypGk+yEq%3^YcXOB{2Y~+m1qbvM|=nq}T zmImonCPz7MW^DPyrUnoJr(X=zL)R05T z*2z?w9qW?XueAX0&bF7LXOqvhs<@3#;Em5{#E-`1ljf|M-@O`EjVuygr#q}RI4x_m z=i8JRIK{WmHWu(i74^uC@rxTJXsm!2Q?X&`5I_+(BWSOU&)B6lWHr%kRO?KCsod=x z8J{V4e$nw_(}k|6vt>?TfDDSUEoJM;tQwX+acbPShd&Q|y<)~%4+P~O^6R4@ZD;oV z!z3iM{$di{r~kqv(#QX161;MQ^VJQm=$HXtXMYQ{tuz83%{N}CQs3i5rbxA|k%r?h z6sucpc_YQ`&Ar~ujPfNOkTT1tPMbs+at`-MgPhr^49*pxrAQs|4%Nl+xVZj z#9hLwC?)YM5Gp0eIeX0(J_;R8>la8!aFUsQmW;Sy%4kPRR>_fAT+VPU2dvTE_LSm* zeEZrv+e4{tRUg7NX<01|Mis?Y6K|#^LT=V70#7SE|7rq5Hx{fAJs*Y=w~!;~C3dXC z&^mm$F_beLxWp*JUSF21qthTk5F*i>U3TcWG8Wzd2;Ru>2tHeC#}KHy2vIdj@E>l5 za;^0j<761C5uCEkZj+- z^jr07_x00i938V1+2037O)-NJ`UN4He8m8Q6mCy!;1#VyZ!gg83|u+TjA~XEUpdlD zaWOF9c7JjrI%>_u+o^2X^;F+Y%DDYL?^o?}b6dW&6e50Ldf(H$ZKXDSc2cv0o_E5< zpauW-RR{j~)`L0Vr6cRy{hV9c^|oVkaZ|l4U1=E3S0sn31(U$Bt6#mIIxB1_O~6n# zij^!L#}Z&XqFbD>YJo_{v63T80Z%6={t6PsZVJY_naGP{y9j03p+WLKKe>WtOV_bc zxp6Q*Q|%q-{(TNl?7tk%)aI@c{5P)!5_B0Kkofzl zG@FHnZXAi8<|Co9!0jqXFnHb=`6s;je7lBdD)40|!2JPks3!eb6arO7Nc({cEdh7| zh$`~}Y{giOq(Z&ZMMh8r0b-kM6Q_c2;%vqarylsIJ3k&f)j(^Dsc)2i0FUt3~}pwWk?RsokTZsP1VLtphBP~X@@COclapRMK4#YuZ=@( zL;qkd*DaMZJ*@dwK3IBMFBd93H$W!zEsT#`G?tmMo_W`4F$2w-DcMaOrypzz>yNAB zd9b`_4i;L-k(%V|4=}Z-NkWvPqh9HxQ=g!&Bk>#9{uKf z1_WG(^g1{|)V|hsm{Hv1jY_YdjC~*1E4HWh_OzN@YE%C9krPnEh>tJ8jqUOe@u*Yo z66SA<%U}KuPRfd$l0Ra|5D@CXCtX>HeR{QPeEsR4=qiT50C>XaWTl47K=ZO5Pw(9hM zrCBGtce5Fw=H|#8^)*t8HlWE902?}5#4YX!%}+U2tSF35HJCa-Fow$B3zHIE+ft7o z8KH^!LV)K-vO4{^qRpRa{g=UT$=z_hy6)8qL5Xnuhgl!V&m(_aWok^d-r~OlF)S7 z)G*yyKwgb|f?EzHlpHiQ_)bu?8Zw%4M}Xe!tB5hCU3rE)^&?`#r6hHQw>-6oWK@FY zp{HU#mHclHVj-l^IK;mcZvrElQ)I$<(a>$bU)zU0d?1k5V$A+zfsQ zhmR-d2GN+oc(qGQL*;kVmU~d7EH?H@NwuvPEWaLXod^-E0@+e!2D}Zzn9qaWfo`tP zEc<0XzkDWEU+hiqe}?(~{2s!IKj1@?^me&|}qeq5WLV*dyL zFYK)_*4%r&og9@bK1t91L=8b>^Mg@Z7C0+D)ivKmE~&TJmY1nyMzJ+KgH=W>su}Df z*wvVcrdQ+456q?6k0cWcdAh_5`#7Oe=_!7{5LPo^fz4x#z z0}{g?i^w(d+~PTF+#ss2Zwfl9)G!~DD# zWheE{e$~_JNl$2Yy-jUp+s{crfmRi&O7r#e>YguZo2E70lu~ElIYey|RQ!Fc)3BD@ z++@P{aR6gmy)kOp0pi9Z1)*3aM?Vs|d&X<6%AYPEI5$L94$fbHybC~2YlnHN zE)=GU^~6eb36%*p<$c`}4DN}emd=|Hu1uqwJYZJa&YrwX8Zgg#hPb0Vz%Q3E!V+-V zBTF=hy1+G@S((WW|h* zEh`B>^ID4F#D&V@^+)YseipSm7toDTlJlKHx4Yw`!qKSxEl z-zz34oSHM_XXN--Pl}(+iU}_65~OW`!m76W{0vKL2!cm8fFX1oSop}ki@eg9q~>8<8SqnTPg5LieK%;R-Z+=yU+b@$GZwt7Co39tpczG@q%A8O}uIA zWwu4N$MNXpL#Qzu!?lurEy#NA&jR1wL_CwDp6vh=(u?RZoecQ?upF1_T7Mflx~o5H zxoRaZXWe?N*yZS4G~V5ZDq!^r(#jhbTV!q{^H>Ufe6i8_J3!^^&OL?HyuO~D^9euaq|vS2dNT@;`-)gmWziol=JLQT})+4TOBC)ir{_0^sm?eR z`ASw8!erLLF26!ES+s0oh*zpR^&bCn=I@h}6YaOqyD zL%r11MnbKWPbd+&#`>L5HbQz3Zp#}`jQGQ9hJA_Afy^;7nl-nF3kC(sRYH_eUX@P7 zQUsr1pwAVU&XNtj_(xk4_{&whf}JbQr0QQ@!tNbTZ+jv#>bC7MZd6I!%hhS+)(p`O zjHOntxCD(+a#nP;{<;=bn|UTk;8Hc))PZ}ImNUi8{`!>t>b0QjMS8AeJ|AizP5aiO zqc!O~>r0QSiqPRdAS*{$?b-ZXqWV+0V`rgrJE>#qtYb5Z-+5{}v#O$O_b5d7Vkmpv z1{1$uw4Uf!GI>z(4l(2_v@7v9H)Gd)97GqgX=m`5d+g5+Cd0J>QJoc59k#BdbNSI+ zYkl8yrp?-^51JSD@iBHWHGb4JTiEY-x0#119f^dlI!Rr@_L>s~Ch`z0_uu>$1p)@kyy820aghBe z{B#PgRTRDUFvScBB*uRi>*aWqS#d>lQEyMG0axDiU_z4w1k3uMCO)D0;j#hAWYL^l;DpuU&NM=!%$Ngw_P|a5w#Gx&Gdd5C-@ZB1WG_H5N2gbe+ zWL$tWNJQ+T#PchyS@;2|i>Iqq2?Q4T$u0`hC(%&yHwFzNVLfGoi~9) z)Diyr+@tM7ZB>E@W^&0(zUaZ;Rl94;7w?!dcskFl-v(ESRL=J~%RI zA%9f@Q5?fS8qq9VxQWCeDXTD3cA$yG7{O$$1b+#3{Le@h;XLg4l-^i72+3o^Dk_w# zc&hlAQk=vJT2|>g$z-D?@Az{FIm!65%d9x1IaDr^g%F)MXG!POq?aq|MtzkuMqP6( z^p&(6KCdw)uE7#}J_DeP5sCEEHM}ER`wGn#(r+qdq(#FTb-g7Pe?Ak?naXwuK0GBZ)a&HDfXY=qns%W0#c)`epFAd1x+$ zeCtxyspv_7A6y@#$dQC{thj`bbX>%w`$ID$(X_!9=CAs;-)*zKFcBZ2p`ScFZ>CJp zC0{>YvwtR}!H&XmpMpt?k1z>J4 z%?@<DarvEtqnML8OYVaIjv5#Xqi%rf^*u6$Z3Z+r@hbfITmzPKpKu3 z;Msf%1W{EWV^d)&0;_rJQoR?{xi)l7+KDXpBOlaAZq(Y4!KP#?6BQsVbR()l``Q4+ zXeuvD$h6>6;Wwl7Q;JdtNy%@%^$sGyt<(w{LILPU@$jnqNj95+Co^0Ypd4)w5Zaf? z;+y)*v5lU1&CCzS1 z+{*iHgH?|neZ{6n_iF>SLG?Kypg*jT)kxl$)o3YJN7yC&jzsvu%T?RZC29rp)(ve! zmqWcozg=0O-(@9V-!rNvVLGy*$XiiE_SI}v{1+d}3B|*_9d;CK`3k}02ilDrN({VR z(s5pqp9vT%IH@NKxH3rvRSwQcvCUKwTI|~zPVY3S2L_bxsG}hn6g_lJinJLN|JO<) zFY0olj*u&yKIww&RYze%x2PS!Ia4Dd!0!RPtV84Y@Ozd)tdZ4mKBtFPFngE)>nz2v z))1vuH_32U0qZU_XBOlhT9e9~Nk(tR3RMy8e;Gm7mQ4oUC z;gLwf@@o$Erc<%z=>MKrL}Ji%?Ciu!SE`3IkD|S(+Hq@82L8hgdcpzNLqr+IPV^JZ zpvq2V^uOZQ>S;W?nzY$UVG^n?Bnpy!TOwo@V7qB;Ged`guW^L}0dT_H7y&h|Zo^%9ZRd@;b}jBV7xiiSjEVz+vY z8@B(!y-E^;kh3HPhQPBQFgoDj1~dYToGQb0olWYaabqllNAEEEc)#OjQH&{^SVo>M z!}zPuR_?2)`|IfH2_X!=Pk!LilGu8mWE40|0gX-1@{3zovu{2qvN%{nuMtWh8-aaD zW(|P=pO3e7>PfdOWLq;w!8w3W_&1OjB8L}^eRtv2sJtn*{LR$gvqSmqr>{v8{TI<1 z>JlA|^r1hK5JRSjou##E zIEYouR-lyYeC& zFYj27F{)wPoH(nIV&`}o=sBxXDaIIIBlHc;><^t^_hXeW9|!I zEAUs11FWVAnbD{e{lB$m^NNVS@;}i90JkjJ!?*4+QkMUo6~#p>=aWR4IQ`48sBdu6 z%}srI!+sV?48MDf-Stn)`B1h+8~fX%S2fz7GBhJYCJ=Hr{=l0k2{$6#;u*erIk_)R zo4g@9uz7OTYk;zu2fSA=j&sgI(&lG?9B0S6;_GHsh%4dw?+s|tG=Ovv*P&BFnQ`+) z73T(?7302Av?jp)^e1-R-29}TY@F%TzT2H_Jo(XVsG<7kHXRLXdw;&CYdd|Kk#QIO z=tnkcc_IIARp%-Ue`ZL?1BBvqVHXrnMFjw5+6VvtrR)6rJQAQkSIOi@W%U2K=^SQd z4E=_>7)ChP`&A7MY0RHF#3%@eys{?{;RWVR1jv}q!0*?&INuNvmI0zakqHqboFFD# z>CtSv+p&Ajuk3MitMp_SEHe zH#nSWi#K@f5n!kMNb|7u;PjlrD=sbhr-2+P4#gBPrFw2jF&JTCn}flt~Bu0qO@Ke`^hiwZRDDS_sNa6$Kt+>KQsKz zXL#GI#yKS15pGj~X|VorX6q)_Z|BE{8*G6VgM(IglifxT--I|>3YB+Acq+&l?&fod zmLB&?*P@h^D^5$5c7EUyx5iTB`f@|hlAWZim0bI*QXs~1W)vVON|1$YKD`|60E?;JPN?@AA(av52_Q)} z?KN=gwRZ%42SF=1*bg&;ZMTyAyo0%OX|1oj{N1wKhx7v!i#_cS~k5*&j#Fsdiyy}r`NE@QN*&R9(WppXSl-P z%Br1chNePkdPNWP|{yq`F@;D}bZM39`4oGJ9A zITcf+bsprBO7jaeUC&Q)XNBE1t;X+>8pEm1)Dp}lnF`4Tp(0F?!WvOQNLcyPAi0b$ z#{rBl^C-Ca(;*EfsLU@JWx=7fPgoK1QVzu!+0=HHG<-ekUid!-DZJ-O$E^F7gCliH zNE>QkRn%b)V;fyx8OVVRQA!A-FamAoWgz7WBe9RONmypSW5V54ld;YC3a>WM-Yp$j z6PnV^3I}DVoVkK>`uk@ff;-?axOeN-jp3fm>W$Vy(!*!5xH-JL-SBp>s4I$_)fCWS z(P&6p)$E5fl=jBUgGuA#(3sa0WCi!e7bZeTA4=jQCJzkgNd}VH0!teh)D%R|=t=4@ zsMjSg>y5&jD+Q8eLnV6wj_8f|z<`iWR>YzCN#V|-J|~FBypP3WezpH{y&sN37f9tC z1fg^q(3=RjG)gvqE=?kAEw0N%qs(}^Qq@b_ekzt(H&X*Y)6AxTCoe4Mg)~(*VS_K4 zTHTVsYc0IDnW4RLPN)CFS?Gg9E##&MlSz(u^o)R~4#P}W^0L4yy!o%D8QtvW-o#cB zL}VpZwou@t9zQ8iYSu&uzCfX#eS261825h!nSU3bh43^HNaAAc)_HAgPl2WJDks`r+ z)sa{dhXP_LRs>)Jwx^7|fHxS$nGum0qtT-LA0evLVu$1lv$DJ(NLW&J zFn?p!qe`9P!0>>(nG?Rg3%hu4LXaM!@Jht0-{*w@1NVTYNWK^?r6QXeS2tUiztxcS>t)jAjCr-?ctC@Fj_4K!eJMkxW%F{8lEhyT9N zdC0{(HLY!vH+^g1U7IWApp$9K);PIc|Y+kh@;PrBhjb_%q-n*vHLi2t1m=;o7 z%j@%OKjA!L1AKLHK&q;3Ep^P&`?tj>`-k-@-h}CTY>Bv z4(6RZu1vc09U-IjLS_rZFOWcU_@V^`9GP?qchXcBm|D-JShlfWgqne7wy3i{^n(*X znzXsV4Q+Zr*0i<24Y+ef5Hoz}0$PnO?0y(n0+v9PA#*^hJfkGQ4RFvQmpHRSKXD{1 z2`?@@0;M^bQ&pwS7%J(oWVEEM7%u+&-8RUB|5D9FVoAlolCfj}9IW@};NFHXFK7iM z!!;F#O8=h_CaSSw47Zi>>^a*-{y*J?K$)F;dnPmuK$+PAPoi%kczKe`>?CNwQzPC|P`KaN*vLvCf_ zEI4^R_@H%bDr}|09AXySO*sQ8loQqPd}x82iqa3|Ztxp640y{bJ-yM$pK$7DU@%Z} zs*=Z?LkDNzf^qt(4Uu6@i-cf< z(*6=h+dhAIlxtr!#K*M-qdLUVrqN{@w0i153kk{{XT=Q9=r}B(F}pt{nXzWA;>_&= zyLrQ3p*a->UHy*ir6R2vuMM;sX+zk*g$|zB(V@2kk7kf_J)DDfY7grPkHE8Y{MFqB zF>(O4z7E&VFNviQ&rIHg%W>SuvYW(i6DYYBA6zEZ6Ydc<=HuBWnCx1C0a)(`4f!sNBaik^YUAjH6MvQ5iJJZb&iyYPv=tQH zRlZZ|Vzg@l&P^nU=QhkaRdj}5QfZbJ9*gK@xqC~Zg(c(`II!d7pDf$hU_nz{I?dJ4 z=I))TY|_ap@m-#y0!x!-fY?@rVlrRT^gf2Fzx}-Tp4(R;TjvHbP2u63>Zr|M;)pyt z=MuQwGvb%`_w!u0^{n$l2pMmQA6m7vwV^{R$gP^;_o7{?4p znNYpF%sbZ*5aGPy6?@+Tz>QNUJmMCRuXRZsJLf@Wu$>@ZJK@Ya1G2PI#JjgH3C=Ia zeP7>wKDK%{1wZXRSM)x&?dA#KAH?!a-*-OjKDV)tw?sj~v;`h2x*c=}$E+owLftRd zDhZGT;b8Xt?BYS`=kR@U4Gx8#-ddInukyJ24e(_^qEltV1w@>zPmP1}gx_PajXUf0 z0=P~>@x3_x#Y62FPS-;7p;rkWAa}k`-#I6FM;HB zT%if<`dN=*940OYsYPZhn&hMIx8G4`+Ws$)tok27@#+2ejMWrv@0rm106HC;QIwk_aIB&47; z)-kTd{%2Bmz*(33E(kqKNRLRkc;}=1wXQfa#ZM^eU!;wpp>u_PmtF5wo>>we`{o?6 zG5#`945$V|g+vB^KvxS_e5%bjAgOiR^edDp;s`p?ER4G#MKqb3_k%`q4^ zSn&sI93)_JHN96?F3C=K*IxHJjdIt=_e{A0>D@~w#e<^dhmx-Kc#BvOV$PJw&n`A-H(Uu1AryY&)T22p%av=$8=sSBc|0)CNjSq)k+G_rmcfU zcJ3hR91gzSixd8Oo8EZDjUWrA#2bS{)+wTyX?~=y&DK~i_DbO}UBZj$ejX)_8_%hC zeMdo>#*tg&eKZS-xc1Mfg?6lTYCL4qjX z;)E|(v{*TvldT~Pv>xfOVCc=+iGyt?yN8nbTE=CN#cBgS#664J>@9){3zh}$_&60z z9F0FbQ0>e396{3^!VQ*{RY1bi%YH|*wVdeh6_rh~wwUkPD@Bp=KrJT<%B*8^r>@=M zG46{zm9|_qS6O#^3J7QCL9ommE#^!stxPl5mz;VH?XH~1C#UyD%Z$1>T-ElrkgDV(! zET>_!s;4(e?w~aphph=fQp7&?gq?Z3=QOYj=_y?|2EPRGM+(6P0p?T=0Q{+?ORem& zFaQm+EM-SVO0Vp3wgFeAjfR)=T;`ax0;faaBhF|7*7Cm|?fIf`Dov%<;uT3}9fY?Z zNipAwHh4JQ_dF$zO@g27>(n3f^W8JTToN9Nwm^a8qQx0dQSHzwitFpasDLrD4wK%{8Uo4cGX~pW!U`%qlX0 ziUW8-G!Dm$+fK>KkQRl6hq-_vbRn=L1gUWVuq0!-RIJf7sKY~|V7vtoXuJ(PX=kaW z9%#JOD4c{;3vR5Dz}z0VB!G@Knz~X`!luNU49ku0*9OdzfQm^|!eg=Ov7o|XMQJ2G zClWLDgGF_Nh3Q0P9zzrx(;==M&^67LPh*e+kQLxrT(h#qHc}bAn^O91lb|ig58@1H z3Yj2T_H7WV2C5U#8?df(}Fd{ia=t_hn>?30o6n3PVMBsI~NeKx1dCKhSt= z7BLboqztGNL}%>!hKNdcL4+vD!oLIlrRh+83WiLR1UqsO8VxB!wRg#FuExT@O7vX; z;_G2Rym2^LN}>C7LmdxRT8MOxL#G}ii*PPm&6xVYFXw*)#-8P`-D5Xaq$GH)BN*_> zT;VL8F3zEbsC6Qdf1oj~#ztbef)=-U=7hdMyMLgrw6KKxyW+}QrrcQF4T##d1zYX1 zs-1?8uCd0j1$1JtRsvA_6jlONlBY^`z2?a|*{9V-$Y28H{etOuJ{as6!%uGjyzK@2 z<82adM7XZjB@We88Dsu5)gGO*M?2e6wZ<$eGyMWTOHY;%NeaX9Qt%@uJ984N>yr^r z3wDtq_Dp?a#Y_&4guR`RE)l!t46CL!)oH~mL8nEZ3O<-@O?icxMu?s(kQl5PEUHVR zkjC38TN2m!4`v-TH1VLZh;lkZx13oLyFAH<%%XECi28Dz|M=ozk@>9;$HEX&PPhL} z)k^w2*WRV1Qq{nF7U53xSBhIpb<84flgSqg!^f9F%@i0wm`T$WfHr1S`m4AN?EuCA zE;ATVwG08iY=sfL52`#!PD4YO+_^#CV}IWa5|`*%(PXArEuj@b(@Y9N$rVi7rmAXg zAY{n(bslE@JABWmN$ELy!a2#b*Ar0x74aO!a^Kr zo?ABWeozJqgpff>xNbzAPr)yYn0-gXo%n&68`v1PdMI`SR#ITZd6r7mDT283h|da( z+_2;&2|mZYDX4BvP>}gu-W+K;@A|$-9|v$R3H|z8_)#<42XFR3;5JtWMp5dw0v&#jBQTk7A)WQQ-i45gAWJk%QA*_g!>t5iaU0 zZ9Q`w^OcWpW6u-T6C>chZ9k!hIdL6#`SWGyVZuhbJzn;LiHd9x*a;EHL&0Y3evw5w z2rdRffwWR;VhAS$p`dfX*ey<6IR<;C>y)BM{Zca-=Ni3?d-@F6pz(g8Qi9a>F1Ix| zZx;7aExwWe``eL(ybgDNXOvX>q9u0#l!ID4lYgpJG@_!gXi~j^>(QYJ>Z#|kt-C}IQVpIhaBR+j;f&Gl1^J!+=)XhUB&rHkv9@15&SY;u)2;>f zr#E9D--T<0{^!&e7cliTk3JX%@E?JhPl0J2O4eb7I-;|pv5s^}=q?Z$JQ1*EHElAE zoneQ$nAI0|&}62cyGTWrNn&h21tJAp7I<<=K^J>K)^h2V#j@yE$+uR3zxSs2!6ahv z>K=^zJ^gL1t$@tm1pRCJTO)4Kro-x-O0xxB%Tjjo;EE6PdnuY)w@nB)hVdHon}DXq z-mTljObSr_Zj(kbISC9p#wkRE>b@Rwbem`jn59_kj1^>w_k{fDgVhig;RWd5@b?Ju zpigCO$c%{TFi=>DmyM>&GX_ z(t_mDO<|W1tm-w?s#U*i#&fTnuN)RqNeVkLac)l%@HxyVq=``*1Uft<)z8He7p!&$ zdTwl(#bXbxPD+)>J4b?B!$uF18gtYxo%M;9*HK}#6>~S(bhCx#ZM}zXR_XZ4@m0TV zkT7GZ_Gh~M`cFaVR=%fDDD)uKk{A_6tKV%kP!U-a=a4T@G|DS5mp>&&P*v5Dn2vT8 ziwtPp<$tgzQDIs1sVO$DLH#mXa4&u4j$D47!$9XeD9T`S3#rrL|+(c6IDky=}$feSs=_;-zY> zfpyzM9s&&C6O~qdY?D8s52NFXK{(5Gc{i0uR{SajGk&oBJX2j!X7y8&HwQA!GExFM z!gCZfh4a{Z+Dy9ec|102W`$B2*9{a(5HWu2qH#1V5-g%q?75?$M5?XsnT<;g^c5qv zjJ3#`Aym<6In!dy=Bd#nkz8~;H$vZ8NHYD78=|72F< zN3>xcUHrp#iyzg-ytiV*nyLt^Tez6FB$t~<`kA#&5mu&v%t z!zabmm<6#`m|zndECdBL+qIv_VI-i3IN1e?kqVrUgu= zuQn4+BQ8XX93;k?6!KOXagn)#{Q@{R(fGN!y?BmH!x@o3cXTJ<0>>cFBQtlJntBC| z%%65D8CepqeNIC{2x4oQvA6?knXPbG*jykCO(_=_ma)s2G zhw^&?ks_r)&U4?KcJ)pu?_&huF`~ zMDoD1+aQRgv%W8T9;G{M$kBXy^zOfksImB;DL}uLOBw1F`X_y3s9_>ztEmgPo9&Tm z75`!=y@nW1d(ucSVPn5hUEIi3h-~$Yd*cdCA&M6eg=Dbto8h4Z%X+_B7QNul2 zBnUs{CB#ubRz15d6UdyeXJ;Z}WoH6@lHdE2nJep?7%1|Orj;rdEGGGtRbkFJGEic` zS&5tm0LL{BAbO9_HI%{(F=|M+Rv6wS7pc(LOvgcf24kr?e>peh|M69Y;h#fnUV~t1 zOtDyjj{kwyH~t!A`^?2;!`YPAU;+l&p$m{=yrhSeB^i>15OlH4jN0ms;JZ@flSUJo z6bb@%n!#^-*h6KF$m;Wm#*GHMlo~zi-$4edh^yOA7opV*b_`0jz186+6-rntm@sPX zOsE?KO3atyH$$slt4o^4Smou!G@XsKgk{%ncj|m#+Um?Y{zjo89P1$!X9I>AuAM#T z975n5Kcy*VE{OjCo}NkB&leoR6cRf}X1GsMeIv;!WWa8hMS(}Sx1=gQ^eW#PSZ1>MSlm%J~>5k>!L+8kAvTSQ40gU1r= z{}lynfM4{E(3E^)%d5Hv;uyG8zs|^iJ+}_wA;ip$!9l^-&FRWvYx7b*1k}^PjiGku z}r`Pozs z_wr@%)grX^K%3f(qvh@Knk12efG3`2?JtdE3Cjr2}0}MWtSyJ>5fBpr++?GVaXzoYN@a z5lpz`NKnS(n@H~Am)7aUbm3R4vo-wV|A(=646>x{wzbPfm)+H6+ttM`ciFaWyUXsf zZQHiGY}>YLed~F#_w(*=pA+X#u88#`BQi4c&T)@<%`w+0*`9ew-$*&ei8WC;eISVo z5!-BYkSUf2!ALd{*&K!EKR_w&Yln2V(QVm&Q8MsA{^rz6uM1Kq79AdZrZQ&d7VMI& zp2abOwsPFhBzX+>;4^|z^Yz4_4zlT`gn(Go6MYIY0o zNjE4d3Bo(M+SoMZ#N@C4uO4-e<6j;%>3(o|1OGf`W%_hTq2j3V&uo3-qMp}a*RCCv zq8<&d6B+QOL~JOYE%_l~b!2!i#9p-8-rlTtG|%E*r*H=Ug<7TP-(RQ;f)WXBwROP- z0pPsNVJoD6$2JoBV-6s`Hop8F+hG4sY$LPge_|VTGO)fswqkx3ACA`PM0$K#Uw6uj zY_8Lj$rk^Yau+UAyt?8+->xQ59&j3D`)argod;50*R)&PA&yYLo1_L5?noF=2T(Lh zK;h1Xh4HPOjYQn6;@u+VR##pnC{>)rM7$mWuD)Beb^ z6t)BRR1T0+a7Ow1tlWp&!vlWKE@Z;&f-wgXuVUG$ zY4?!5o!vsfJrg}zRLrA9$fU57L9BqCRLHg}WzUP+PE_nF(igRinPrf@Wuc0GyTIh<3fGD8Lk8)|5*x)#jYx%RGa~_6msT?y$tZ$P^rDu|;vU_w=2S1f z{WzINsMz}BS590O2LF^~0l3}_ThErGwX#(s!^vi{D{oS$PbS0{;-S1Y#h-mlZhrVDL zt|2beb~HfP*UrwxUGNUOSf7rx=r?N`sJT541?(Ig^s~nC4@7USZAWifuFSYzIlTLE zdjQp}N4K%qRwv6f`04Qsl93p3N+O@> zBF(vsAez_2=2EP0E`C(l%ZxS1 zWG!=2;5Y^Z`?GT3^ZjrP;G(rcg2yxL@5IJ~N_72$XW6ftF*ifcgmYUo*d z$c_@nPTGYWYP+fgxyJ@j$~lPjPlD3GH8amKhRrXAr8`h@j%2Xr7k4STx{qwKp;_&b zz7E|vOh@ka^9v~(`8J84E11!y_r;X%CSX&Z9nqrZ5UnpQaYDbR2sB-uQICa~U9TRf zg8e3N2&O3W{b5Ajs=oR&a-#AW;J*kF%R-4C!Xo~j4{CRj%nD(y^KUr&kvqblIzT+F@@?ohxbY4i`OT~mozh22XZ3~Hz=k5^b&;e`CRDc9T{ zqo$m8IxNTB1=-h|_+A?fc5r|CgPPZ#l8^y>$V8UX`5qvMF$8Z}1x&pvNR3qgCx|ia z%hRvc=mHk;np>eFg--^LeJ1I*ti1nAb%T57C#@E3VmWd|E)HB`u<{Tw=A={*CJfs@ zVy~P57!N9~hoht7<;N?n#Y{^c`BDA)u{|vtF>!FY{QO z;Z%~=`hjkDFM%OwySMV}uThlem|yBOv#Iy8V1Yk){Jeu)V1$1lg~xGn*^HxmhOGNc^U)dnLkti34|+ZwpdL#w1ab;1+B&m-d~EW%?+W!?)RcV+bC3vpsUT=dUCTp+&%Tr-$XXX7KiJ(>YA z#*jJCbNAodRJ+8n{W8QM6lUSDN%0X0U6p8XFc>{qLd-%S@zGa5@IGKWNc@!FCfUw zJUhN4YN;CFTzf&Qz2qAER};k%vvV%T`x;!arpVlo)o>?`VWR#BVr{+sZ=j+!-{@~Z z1*|&|sCXlp`8!ZyP4S;V#lYWziX5;)5n0UdJF>@b@+b4_s*8#89KX`jWfE>wFv(8Q zN{f)<555y&j`umeen%F0u@p;W(XNdePI>mH6sWI&bf>Mx>#t~$PbwT888|gLl6Fa|3z%rgs;MXCY;-wB5>4&ZT^E; z)?pdbtn88Aqe{TYjFVKcxl&24kkq-!FEb3<*H#?s!PNK=FbO%g>lt?0jp|q>yT?s%)zY(^8i#Q$6d|IBQ)UYr^#` zI~6Y66lz>yq~zMC3=v!zOmFPhoA$y!H|Lvot7U)EseIC~cIvO553}um5aL|{1YlPd zl|8ssJ~>$*?`T|6%uVOBHK9;cDhs;&YSGR#1#ezX`yS{#i|*N-gxG=GyhQD)ntA%b z$fX>e)Ha$|`&6!cwzM+B(6mX|w20gk2C%#coEzCvUdnj*tgO{7Z)&x~*GT88TjZ`$ zc@`#}T2>$*LfU;~NA9quMV4GmfC+^)F#ILlVn)bfMB)`a@60pPJDS!>f)zHXO@a&? zY|ZzWo6`Mi^^mv4i^qMGqNvhT-;n-I>Etc#J?@BluQPP-2*LgoYG-VxIdkf-;nT}9 zX0G0+=U(J2qTs&gd8NzIHzPLhAx`@15<7RCFWZVooE6H?!znsn2BLA|+{CW-H8nOj zARtE>Zeq@%)x&>lKVO-iiEgOjC;1$)l=4u_&JMfIG&j$7b`2C`Jx=a;L`g1t1SEdmxG^jQq zSX>sDr`MdmJ6OKkNxfXaMuN-XFjJ^s=)*c6zST-l zo-4SCx^VpsG(*rgO+P1$DUr`*^2(Btk~`u{B4OtBo&$^E#wmm``Vywab2SDY?MpgI zPI6q>?&awDEP9$!rN4>kSmKG75Z)F3oUnrBy46Vk@xz;cYJJf>;d@(c=_+~b@5b)s z(wha5L%0%Z(aLJSvh$&+R&eUZu%5fr6@@LMc=6fhvv$7n(FvOJw4bP=QBvfv&W$Ie zNHGstl}ksVy*L(K#@Tgd-cKs=%A$n?wmPTAD3G0~GVqd) z)GuU>WvV!KUk}hPTumlBFtDN_Ga70F{!l%5=xO^@oRbb zsUYD%l%mv}vE7bd)A?Pp>t94GNmy_Y| zVBa?!rHzm<9!%leI#@G!c`4w!wbtLVk@qd8p4t|&d`4qy&p-!Mj@NL=wnlAujQqQl}j>gB!ohdt*Adp$us z=ZF?@QN)sTOgN%q6vYQNJy3-~(dn2fg!Kn-y=~)}emG}&GXn4T(@*RGQ5+PtcES9a zK~x%bR@RvB={VA-NXgKk~(ZiW3tjx`P z*yX`V(`sx^y=rV0p;5~D!LMEE^_9woL@xD{+sQOf?d`+*Bp+U@_#V@|h-U9zS!wDO zvDs6ba4NObtkx9(TbVT8nx-s8nF!w{ZJ9|-kMlh{82QuhRx{^_BFsiKXq%08Z%nct#nC^GcrgB z%5Vo{s8633Kh5sVhks~5UkX1WEHX@Jc(3c-H!@L3(yRi%O|JqUc(yV5f1Yiu7RcrJ z2-xD}TYcCFXr?W~Q0&`%MNmX|UEvXef^3YJ9{x9aZ4JdxPdJ!Z76!heo>UAZudNM( zBXQe zK2pZ!stz=*+fVuG;(+Zi162h+z;@5b&_s9NzZjF?S|DTcBeQfGEVoEo@~Y+^l7CR! zMjgj|A=cVD+I}kJo8{m9YKQZ0cy(3aFJ3>09Nm`SwfsrYXER%!z&7ZK?RJTQbi{pi zHK+p@7zaoHwJfS=(muX_+mg-3Mcu<(1ev6AKRM=~lIv!Q#JUc{MweBYZv^CuxJ0Xr zRsE3YC1|~KQ;X7IlCG!5t>;z!8tTzW)5d#RK>6?IJLF77fL_aNy&(J#`R}H9q5e<# zpL>(o{{TLF`dMA~uJ_GsBXQ0@$1okrVfiwVt+qg6JfW{3nFN^0M(N(R3+5hW5=Q%R zVSxn=fezEGRt>W!{*8) zEq*6xkSlweNSyn8O;j|$6s4j2QYlGuV4!dn{Rc)VS`Gdyh~h!8DT>F8dWsBfX?mZn z3Y8?`n(3@>YY*q?nY|X<2V;$>hA(0#7_iq<4Lmaqc%ha#F`H*+rHxlb|JWq)$NA7S zkv&eg{3L6jsqHd@e$6|P(W^+cb!Ius^XMr zI{j;(bVQKu@%qkt;|0olUB5ceYXs%7y%mw9K=T{gwxWM$eDwZ$x(PZU0Nj*w1MmDo zx~P9<{39Sa1iC3x+98Ycn38Zcz4Gi#W*_akg8n}@Oc{1xu0)(K$+ojyt8?oBWUfV{`tylFCw zZ3-lagZ5y|fw+2-q)WQ8&22G~0)e|jLAgV%0M@^I1#f{NS45qNFE=XluM?{iE-0k_ z0OVWv-az={wr~;wY+{CIA3IBsF|Me*iek=M{0_?ev?=8MN@105C2f?lTWbp#)d>e}dcT75`pE5aFC? z{R?fM_(x%b<1f^{js1bSZ=tsI2ofTqT!(V>KOy&A0*g+rhN-5m3x65X4TmK#P)XPd z{ALx(%Uf5@%kM;+B2sR@dvFZ2UwRb?t$r4{E$UNdB%?&+$JS&XPo-K>aU`Fflc{#M zhqa3iKL*J3>>xOE)=}BJiDl-}%I^H-5k&(X?lKC)4(g}SP+fuUENYEdCr(s?mrECs zIJH6A-_P?WBicdFlFRDT!}zt11Kij623RsR5rfKI7q?FFez9_2%FPZ_;JFQ_%ZYdC zC%i{TqMG30Sp&IC`hQEO28>$8CagK`L)ZvqYP?Jfd)x?EG2%kL3X<$2V$o0)*(~wE0G3EW9$a14M)}(nNLu(dC z=lUrkpUPM0Q?f^?`HU)}rMEi`zp1T<=X+g>Hr%Xcz<)gL^cfOO;`%H1je9 zuay2wPH^HK`0z4J0-!%Wa#I0U#Ab^C^J)eGzdyzIQ;LKu8$Vwtwb!$W(}{9*SB4GL zng+6{TgKGtTRVl8G$Jhvd% z#!ufn1TIHrdGs5z10P@cF}h+y+=8plLpE}(+05MNbQNWf&s1m;M=x^$a6OZtx zX@Wa%+0Ue73TZJ_>Q$}-G@5>_?pWv^nbfUlZ zRHL}Sr$JYSq=wCwiV)=KFG`NUOQ)1sj&MeiS45|*Jy_@7>lUi3of9LP302IVj;LbJ z_vd|nd2HI~9=dbmbVf#edGO55OVxRmAdhAaC(qNc@a7v$Kmd7)C0NqXF;CRQM1P?P z=b77mjOND9LizKW`!QW0{ppo)t4tzinVcg^oWF(7x-%2p`4kX5W&Qwj@-8O;K(*IyH z5xL0R4U#IwI)@uKeePKQ{iU)W7s~)=Pqd2hyK7u^Y^%W|`A~O7m|uBePr%w5%}=Sb zTV{Pz$wFH!5mA(y*7A(2c#TZ{2<5y*x>9ui{8hJyzgluuTtXNwRVSCPMaO2SOzHYd z5A_4a4?2j~*YkvBpfwE-Bjd#HjK#?rzsZT?;VW!mu=v@}^I3a@%~u>t`m26Ei`vr0FGChA(Gbs53MO1 z2u_^MLJA0zSWXWHdoQdTon{32NA{m-BGs$Pm2kvpN8 zhTmWDr#?=TWizj190$nb9uQzHIod{DvDC133T!`;Xax3$Q}N@&3x+jl6$^Id>+xa8 z2=ni|z1}5ehZ?8#J8F_~bRlA!1l|T`Q6?D)+@(TqU~)K%kwV zZMfl<%mMm84+i{yTF6NE*1CEvpnj>r~8_bvJ%$g}Wc)A~^)c}s`JG1NSpD=cr zpI1*2Z~Xnexi^5;c)3adyoo7ONZ{>Ipf(vJAvW(Yl7X!j<;{mm@!aSHPPi0-#SiWD zy-Q>Ir&zk$#H!xoK>EsZb%sj~IH$07)4T=T~2QfT`J~ zZ%e*hw7GE%P0$+RgQY{yYbz@F38>pU+%p0i(*!}od$6mK1-uo<a?(TmR#{ zK&@JfvUu*~jie9(c8-<~WC5J?_T_^ZF7!ujtJ9Z+Z^U()RF?K&p&4dV(+l)&K1drC zy_vz~vzRj`&_f_Z$*g(=ZZ<6^F$#PET@5QJi9#MxC~^ zaJ!fJnxXESF!c^wKBmrZUs>YvHp{rC?gTM90T^wguu-on;;x_X}yt zmdDBY2;vB^)U!jwY4d!^5M({`vD4c_E0~fYpCb3g9Jb`WL5~vPv+6=I4fL&|m`6z= za=`nt!#MJPW^?%g$!6IYU2$HgjHwyq`iO>Q>Bp?;Xi{1AH}x)RBl0>YFluG?paFX@ zHoJG!vcB2~PN^L4;7~`c7W+CSy*eV?{K1CAeon$Z4?1#aKZs3;T6}meNxB-!HUm2J z%Oew!Q6q&k%UsC4?97aN6CdO71*Y4(`2(@?voWJ$jXhY7xGy`R*c~gvv<`}cGCV9M z^;4hTTjg)B)2IONxW|LJ#<=*Rc)>!kNp3R)&K-H|)d@|fXGJ}H(2bs&vLakQQ%b4G~!J2Ykt0tJEEa=#Q zrU#v0Rf4gh+LD7QOWZRK`Z1Ueug8}Z+pSdpWm$Il~~zS(Dnuc4vdkq~)4^>RY%Ul{3`OQ51=30nhIQGBU_EmFcOk|j6Dw++yq zxp=TKG{~$pYQG(aAB0g&fM&p;|>6r}1Dd91FgDy*{xj%mKI#Lp_ih^Ej3>dy~VffUS^< zNP+^3lw&xhmb~KDsdZ0&%JlFLB1?uNjAl&cd-JPF<07PHNdESrWQ z6M$$~sRv68g)oyFwN@$`jW!>=K0zO5F&6R1uyGPgz>hU1d>CgQ)M$YRk3UN9cEsCL zs?NntRBN=-GS9@EslCDa*{)1MjmV-tP^%1gQ{}I(_};ueAxk~K-M#bq=>uFj#STB3 zzr$MFIH{k07SDTn-5`~?gh=1LOFp`e*~WNqVsY?>I$8u-N4`$RR3E9gn3udj%3EJz zml-GNU^lpF%iR&PwNg@QIRy^vSkR2+fo~RtR>$}H@;N89htR|`T0Yrm#8}-HJz0Mw z2-zG!2C)bac%B*+F@1Pzt&k8Aq*|0)fmNEyHNA&1T#Tv9@9vN5ZYZz2xe*r&;Z6Zq zr0DZ4Y1Y1im8JroLh^S%2>WL?NB#g@Q5syhoimR;k1ZpeQ*R24esq!JNDSpaF{tg( z>gL~hN@#{iuOZ?+V0&GR48bJw8W?8FYwCESpUenMcaL0!GK~$?NPLI}&L$B{k< z=Gk_eOXw?x83I8_(9O)09|`8?GwL_`p2xcyO2goyP@<+g;xaT41;4$Yk^T66LkjxS z0LkcZqGNRNA7#mq%+aC2RtXeo5OY*;Y&KPB$4+S!@dV633taUv@g1+_7{B#>VcQL6 zavFmWAPDPmpgrmv1O3qj&FJ6Mb9B7W1NyZCAJ}i^!FD_qcPJOrU6e`tvHV4%-e<*^ z>siv`vQfxh>Odxm|AqSv{H=90wtK%TidSlgo|uIRI-tG}2wZ)tEy5tcXBU)|Y=LQq zXI%w@fqBCo9$1@p_dAYQ2Ybgxbe$XVjTEpSxx-B<6EX)b2e8b)ZHUz$PvK;V`o0*c zP_A(qsTwahGbWp`f%rXq-fw7LpXE4SuN_q-n%AoZ4!A|<1YkeQzYhPQ!on%D`x1Bn z*(s%ZU_>4k@d{w@sJr2ED&)p~eABDuhX5~*cBtkD@%+$W7O(0Vp~+9vdim`LGGC)) zKQrFDLON}~>muFIWTvgWe8S%S6ULhIICR#Ckc%WwhOYCd(hq@8$LO(GCFD z0UduGhqv}xdL56@@^OP}GE~NL5WS6j1NdDwXSc5bmTs=z%O_rzwmw#t}FeyH`QqxZn0@__KomA2$nkPu;UC=QNp_zMTSSwnr~AG6Nt2$$Ta^&k=p{I$v$- zf((y;>HGHp*1mhDYLN|lZrFO(={L7;UHWa_fu=A*OffHnC%<#d1UhU*6pCS|RRO$D zGxR3G_~*H{ASWvZdA~V7c7F=ad^YnlJ^?nQ)m8yQ6kl*Oz1p?ywo(&t?)hjB?^3G0 zG`=>>i$I|>@r|}nHdFRDFq-m>-F)sm#K7|qbs>piGHQGY!pPPnJWI3;YEkZlW9855 zuFj9W7k=d#%PxR>gRh+&4}fWXjpM3&LxM7dEeH)4u|_ujgv&M4@ip$iGAy``vt&j~ z*C>90SP}O?4M#|o)@#oRjbe)k$J*7qNtMnoE4P)0V8cpu z^ygN+4kUTv^X>w+gHvci-|V3W9Wx9(0s z_3x#@CK|oz={jMls=B~YDK#kjBGYuJ!H`|8FWoQrNMg{|sAaR7E9o$4PsbMvACs1O zxYvUBd5v~-18w6*7#{&sEO6x)W|r(KladSC;}oa5_39l1xSjjO%66ITkiyAe zwHfMgeqLJf?O0{0^y~dKQj$gdTPCd`@JSr>zhaF`egU1NBp2gmr7Y<>OVYyycQwgI zWhc`1vfMH8B(p+hl|~#ki)^^dakglUcHe}maA0mrPRxktGrkC-;%lqZRz2if;TYb+ zJJ#W0$PvRI32bCLp5j?n^Je!{N%%Oo;$%XHxro{!eKUrXpdNWj2sO;k; zQXpPiT=|a4p5==y!`MA<=4yFY*vg;n5-Ly~nlHE3DP37CX}Y;b8kI%GljtOyy(L*` ze0D?yP76GdxCrsE;LsQ@oaT)TUu^ zA1JEc7ZF_Z{vy^3HT7hU9_MXS`ey57A4eY{;{YqVb9TpuZ886O^PTw1_mCO2v0*lX zG5AQ}yI6djAXdaGuTPpHPIp%gjEbj}(AI z1z(Ut#7_wb_-Tjyf|kbEFf(d=983`+>q@YN@y^&=MgCR9<&^$YmMj87ji~VWPaD-d zgzRw0v)%C{OV96}B4?6Fu`!yPY!U!#SUe30a$HRfc*Ur!ld0v8Ad*iK2i`$Yr6tH? z-bNe;l>M1Po{gAORn=7shcZSMK1m0VrE#PP+V<>+NkcYa8rKDbrb6$<5}tAP;A}Vj z!?SMeZsA0>KdSM6M1qYKN@)PEqze7s4@i`;18~0*+X^ZnR1ytO##p$iR5JWn|(TUv}RIgoEP`IB5HT|K%HM>a+}ckn7b)Nsj}2)N}f zp56MnbP(QTwS_U>j5PkG4m@~p0Z96`p~HS16>SA);DmNsAFpgMZCz;)$fDZCY?`SxWZ*;p@xJ^zJq*DUmGF`rJegN#5~S231n)X zXxn*Ffz9kwK(n5^I?ed;Hl%*2tuE~dr%hu;YWn=%@HFI#n0&MRPQG@(B1K2JXc-H) zkG;Hp3`}gJ2Fg8HCy3T`Dy~v7&vMmHBA4!-TpEXt45}t{z5kL;?8mg*QN1K0Jvr<;BJL_0Z{@Ryu6KV2S){JSkWP9jD zOXa?U(0$WK9B?{bAX>ggf30e#sQ1DiZHV}5U)-))9MBS$vPc( zXdh<9ScXXDG{S=?yojhhnNjLRN+{KA0&48NzXV+#K29uO=zEH}@_Vg(Uiow|6VV!Y z{Ip`{P9JiPLYdBU2&r;uJxSH?a&wmxYqVOOvTH(G{IaTP^k3~$6%N%I8e4voPXuM= zyQRwCHtMUFr!Xk02^S@-j7V^r*g(zu0zWuqkvENAuROVY@~Y5Y2wDSMIzJ^7?s)Sz zH3vcPyT>}F=m;m2p@!55$$?xkTR zFcFdz#G|bnAga1ZpG1F`YkUGnt4fubu|-x4(5M>837w) z9$YZ0bLb^=4n8sHjX)ny=ok^wy>C2`I&Qn)B|Hjkoql3U^*&N~qaBi5wUtIpFN{n+ zfkBRsf#Fsy9w{R2g30}|Qlp2fBAE!IW594N18GEjEeqDzo5`SyAu%_*XdJJpGeLzl zLo$!}Vye(k=96;S7(o<+rI#A&qQ^aMj<^Ymu$uCxu9@mEPs0OrzePDoCQWf|`K?|A z0cSr+=lg8OS5FIsrR1&B-!XiL4i}3iE|RYK99@9)x6EU(Z=%}sr>NIyPyA5O9|5~> z08ej6z*A=6ndp_TQOwjnL(pw&wt{i1u#pwmMb*TYKlr7iKDVs^_O>*4igM#k0YYHu z3R1M=?X|!CxJ`BQ1I0nF^l91iC;%4KZu(6ibMCk!*aJ!1hDOX5hbH$!Tsxs%QR@&1 zoT(TAK*Z+p1ne9V{!Uo&;$Is*^#Sbw-U6`-cgl~fSSDiYm3(Y2xN*wax@*#jNB6|> znLH0blfsw6?SjxP28adZeFE#Wgt#BRSUV_ZfT2|%{k&%xi6@dQsFDc2yp!6t<9BG| z20H8f%4fmKbFH=}%pL#rt!St6v13+Oj-TIcOCz6?bZWs=+z-q9GCa2OTfNP40c-s4 zXhFCi-Oavo^MH^3nXVb2E0PuZIYl9!jX6o73Fz-TQ+jO6N3_T5vU%~56}g2&8!wYZ zdqc=J#G3exwiO4}lih0x%tq)bLJ*0j70`MKb*;~0w{X1?ao|{r;DmDTYT@ps4ZFrc zEJoWWWg6R{emR+jcK}Vj1Ku=Nc}fbcZ^WNuj$V5|&O1w(-7!Qc&WMz(H>su7?bp;M zOR$@@uwO3w)4c{lTq(>U;72S!;l*Z%D)|GDL*3Qvz8z9J^FjU!P8iD=q{J}iw+G>B z5vW?*2zT9x>5YgXT3gyb4eBu3p^*AD&KV%<_$R*?>f2GBj=o7CHxWy)Ko3)@|56tq z%{>7WYLJJv4=P(foR)L>`j`w-&Y%HQrkydxIB7XG zM@9Q6&UfT^!{1-4@3w*Snz5rEMKwXBS~Vy)jI$5~j|c}kU&TwGzqW_!+WU9hNB17y zQgML)j$bHGr3-!!;fA*+IUL818^;D<|71tM^NNjMHXcu}v-ZFn9jdw;`!V;B&5G{~ zwbARt@PrzF`=d+;N2MC@7y_N&+4pMO;LFH(*4+-1EB9ZtaD&M0 zmEld%g#*1ZnCjU>W%nN8B7*-P)88PcMs+BzPz&+%WJ~Vd>yKt*VgH6Gi%Gl8h@ng> z|BFyAwd_p@E+bM}q9->U*0h&ari)On|7SIP6TekwI7^7!O_njx_x>%KU)qN;BCW#yBIse@YH(Y4X%x5G*`}?gZL82)JjbhYlA5auuW1Qr2 zetHQ)Gd}gxK|;&)R-f)l&8Y8J1qkY$lcv3&*iOx=pJ{Zki;2kJz1!hpJjnc3E9OpMAD9dWFd5|GR@ z+*(%a&mG_fCe4uIMkKgc6r+h!i>-NfcZhR9aNT5Cwb&)$=*Rvm*x~Ph(sMsM?!-)v zLIf?~Y}?n@ZVry^*0>v|V=x?m?#sT8|4^~adu{su1Q>Jfy%l~+v zZ%;xP3C3h(fpJ;TD+^3=sOXbVzxN?%+xWd|$Ly(Q3a6rbmt^|xrna}#{+NIJ0mD&r z6dDEjIBm18bP5HDWQE&;6nZG=9K$2XLV3-#+wMn_u!GIOPft(Z7AXTp?=@gW>RA|J z&fp{1A|zo76@aswX*Us1-JoC0@HPZYQB8KOXJnzSV!_U*Zf&3OpvWv?`na2#y(sE)a6AKtKk zLYoR2MC^zXB01_56-~3J4hM5N*d{>Z;3}X5`+jCLujo`Tmf<+H#bg1oY=X;G@wb!h zPvpY5im}mIvrSr?H0yW^TJ_&QlV6fpHaz#GzwBLwN=q17SaIx_W2d-ZEgp5QM*?|T zatf56>okmop!Uq~`xwfwXxpV-P)^iQ!A#nJZh^UN;w0ITYpLA%4%;HB<9~9h^LK*_ zwNr$YPmBipLGw=$wXx*PAEUa(aCqZN8$BG9mA{H9>o{{uGk?(JKv?PO4vV!|K#iTP zM2;rqlFF~obToT<6GVlPc5lGMUWyiO$v^7>S>q8UO`H0F(FKAe!?(*ck7ApJJo%ZE zUE_w>2rTc!Ke&eyRr>QkAJKS424Zi#fEl&YwknKxO}`}M`Nh&O{-`n{^N3#)9t&|M zqk@{I8Y3p6h7>PHSKRlt$_U#Zs=mKs5Y(F2T96K%`RM}Q#hV- z@yEmZ zBdpdZ@8|js6vu3uDVd&X82fP7r@o(SACKM0PPuZrvrU#E3WF^>lA&cDI2|@gs3uhU ze;sPa6n@FgGL46|NxaJ9)n`m6&C^n7&{0I{@19hU7pP9(A|XVaOuX*cR02-^V1=7R zNcbX6q1atJg{!4FPO?OA%_1<|mE7=@RN{wxwNQ-+l6dEPY>3c2HGBwMAxQ2Ee+N1G zJyoflh|jks6WHUl-58iw*YU*C94ar6A_l)zO(K+If$!*{P>@7{xJi_ZTaBc z-G+>EBB5_an`Sg0S;b}`+b^_x0hrG;ZzBMV`}TY(Ob=4;s|TIZqSaI5?Wb}ny!qZM zxdUFL9)g7dsSjEAQ{Vn?+3g9hDSwhQz>IKG3VZf+;NE@DKP+9>n(FwYpX_xY%j(+= z3gc*3lIGkNUaDEmT^wc7p9e-)&YIg^4}!(Te@LU!f~+pO%r1TEg1B{uwP_^uK!?$y zB!fcTrQ#q!p)(;-iDCNTFN^o_eGwS=(cPs43H5<^24<3Vv{;Qsif0~dlM{znLlT3E z#$+-w8W;YnR|)P$xgkhUw*;P{H&q4Gf(BkN&$<6?O<{qM zG`FoA7ozA~nJeH}2}|nlwi`=dqO$24!el1;NLBb@8Mv#AzNtIcrT0;7vSCW2HQj?u z4&OQ(`2$^j+lE>%e?&?4Ysxs|7d?wlkbQ$22ONzdlPa7JN~uRH=UCcM>Zq<0UJ1&a zYHUA0;={F$_)#4ea&(BpRK^c&e``oOvhSBYIM_rzoDH!9u!?o5&y!EIrPKnhKg3V@0Wa`{q6YL-nu|Q3E8c;)069KHImx&$jBc@8 zv-9fcpt)3SSX#)-5VHJ9 zns2t&;f!)H2%q2IBd-qE>C#6TMLVcuxY9NzYv1cxT7Z@vK(ep#wU=Jj)`od2Oc$1; zZtYtn#9RA2cO;`UmIXCzcB4kGRM-FyJ{sspZR5hHRH4|VvBs91rhp&s#bvAmB$_f ziL~+;*JB^E8fpW%2dZ<^{4oFVQ!mR&cdU-n%Ew5VU8PfWL*V|^$SY#J+VSnxqUqBR znd>Fi;2%|`uUB!7&j*Hnjr?1(xyN-62btq2&bc24_3*YtO@phiw@?nxH2UY`co1}W z?Z-!+XhiVpI~b%)*Q3ENe}uyL8qo4~IJu1A;~MP@86q5;{^%Y^Z;+)(CI<5(`r>}F zzMDUPhk2&q@$3M3XIr{V5Aj(=Y4yEM&bY8lUta+4u%be=k3qC&(7C3%RtiS78t=f5 zk5R2QbfOt~B)~Ks|F!~VSNcgG)P-I%`i&m5UI}R%m_5gwJI(ZYn_KWuvgLV`rwEGl z!vqSJJ2KXG^PA2upR)Wdou+b47<{RDqfZ>bfXshec#!4x9>ap!;BgsAwJ*R^bG;ZeeWh5 zHHOiQ;es=QdOGKNIY$n6bQ{tUeN!s|*@W{Vq!(e$A@1NcVaQ)?2s&8s*{64q>tqjI zC^69%v|4VUPr8E%ICD~?;Hq8Rcig1vg}g&1zl&5hc@FHo7Se5zf;DNyyu!?YNJLX* z{p>#U|0C+1gDYvjFW%U;Inl(nZ6_1kwr$(Ct%+?J_u*SVh@^DFtWvBm|3(&NKcu zYFohQngITdg4!DGegiZsR%j^=(GgBYKroR6UmjUtc@=9IYy1O`OyRfMFbEMH^4YVC z()Ho#LyYv;%!b5<(@hn^jWp`Rki{C8ZgT|Hfq|YZv%p&wXFr8vVS;v!n2#V+V|?#o zp)H|WgGexu+E7PnuUbL9y={v`?`fgU0Qjle4;DO$ zl$8Dqju{P(p^IX$wZ}IWG?Vc`8q@rG{UP0)z_Z%uI!^RuykSa*%Bn$!L?(g|P{4q& zp~tb|EDaY{f@m~s$t0M{#=4MKgE;MPjVD0s6T(*P6n)$?6jB17G3K$cghv*D4k{cN ztuIL=()!Yx&Is}CpD()rmt8v$^!dJHccG4Ut4IaaU(f~Pt`O8dM=nVgmlnlRrOq@rP2?zVccb6X$>0>HWs#T z)7`fs;%HN5vPspW`|kWqJk*DJ#0tsp42?LAFM1Z8Z*k5u$pi2jg#IDIAc*pMUT} zm<8>59Dm0(H7x}60E}>Fh3~!yD}G9kNq$lEyCvlJ{n@WB|6lq%^+xKE#mhlGTE_A> z7?RWXsP$81q?YK5oqE4~-TY$y4SDKL)4@|~&auQ)51GHc%#hdor1nSl8 z9xy4b8=^bM02{-;T$?PY>ud@BaXhwE9h)9R>-Y!fT*+H%Oq7#|V ziEf?E_Nn5UPAT3!Rk8`e|K+z!$?(R#E6;R3G~_(cDHf7}iBY9dNL$lD!c3z}E6dWR z)V(9x^t(X;*HQyr;Z`Dd%1_W%j+Q9{5qK{P2Z#+EC@;7$4%@#L$7z@7)Db&? zQ1x^JqeuIpFO0tsP+E~p5tNPkhByWmFMi7em6fA?)^Y8C3RxF=ow}csUm9Q4utRC~ z3a06Fre`gvH3kKqQwv_>=FKXBc3?bqO4C<@J$J)o2q0@SQ4C0-@i638yd;v-sO3N>1@;K35w9Uw|Y*8YHbT>X!f>DR9zFh{ILJLDuoy`J@!dD_Mxkl(^7_5BS&>5+8y z(56iYSZ^r4ruXQP*3Q(ok}$eS9HiJy6t8@1#S?>ib7^2i&{O)21jNc{IJ`a+K+(VE zOr#&1O15vxwZF67i32P61lZ@(HT@aZ4F6=*=jZalFRqozhbVZyv%tN2zN0Z940C{~ zqN0Ng+7z--w-;vxbh4I>T{FPjLByk~j`hHvQE*gO)!0zV`H2sSJqdyzLV zA0StO7Sfg36^-7HU1@t7kL4VqcqzubNNAX&V->?GAOWZT~HU^qK=k=-N51rp}BT8p0-MfHMf> z3#UJt`ejP?Dv}^s?m|3?t!UC{K17P?)P5BB7E8=&0 z{a;(&^PK0C-$W!r@bnU(>4TqQHV<*O<-0e=%;77q5uXdGv;{D@^v%z-Oth89jHBy7a}a$8xIimzb5`--L>RGbzzDNoNDn>H@I(_9qyo6W9;8J3<>HMX zLj9d(rUIE=U#*}&CxoOZf)D*G)k@h?=kX4BNJ_eVi#q|$0oYK_?!WsnjOv8CtIssB zK{PYozK-F@Tjjz5okM#3ejE2R z3;vYw@db!`>}j+m*sWaublu%4b(QscTYGFr@W$}j)nOc~Z^@?upF@-H8~KW7^hYb( zdM7u;pmn@m+^2hIJJIa}p;)?nx z!0)!tEG~)o0jI+C@RyHO*4#({IcR)~yRVeloXt^)C3i3jpc4bt!p+8fBVq7KQi9 z^4gZuWybMsRc}xcl}ovUNM))I)<%$4y|cCFyiZ?zmIO~yHqTN^k+U<@e3P&;HE+q$ z_cL$ai+EN`Wp5xZ1u4~EybtO(k(RTbU%Z-UYhkRFDY?f~yz46qL@{Q$b>darA&FLL zqGu3sh_z?{YL4j5_Hc~QFEM$S&zs%D$444e05=|j^XH9PZu?y?K|_aQtjlU0!iP_A z{SLLE57MQZ|Bn;$Z$Yrfi|`Ovup&F)e0BQ@U#gSzafE#C_um)^gm%FMv`UMow*9hT zHJS>abTd_FfA3kOsGG(oWDhpBJ=Xo2x*VBd+Dc#J`(mRDLeRUq-!#}Qpqk>{W>;#O z1>4~fC@;SlGpRtyu2)9nI8vQSxCwDPRV(Yi?77g7JgOue#kCvBlZ<6&qvw;4lhwu+ zUD|)yIh(+z){6zr@;C$29IhE36Q$WL4M>rgtTr8ID2b|6k7BDD(}**lAYYLDY#&6K z_^_!$)s>j<1DYVb3BuVKvP7C?DnU=s?Vj20{v(|3_$D6>297*|{DUp8fIaOUH~)Y= zk!Sh`Yl?_51(9>)^;&%hsC#@Loz4Cv9DF!A!3F@%ZdFbI+Y`*QfL`FbUgt2YxbJ?? zN4Mp@+O5lvYty|{g?*3!k3uU}@Mb}%NPDQ?i*DGy7FndP3x*aZuS~yf;U0Ft3F+ zSxruA=BYZCSZzOuJZi_u9PXDs!Cf;WZdE!jGMxtSN!D8)^Wgn>)VUd^ch~n+pwAla zHL1PD8hCSm&RRIU2x4C~i8Vx;{<@7X^v02Px$;2^HXZx={YqQzP~*PaqUnJ*J4BwP zLa$#_o1ws`b;&QURvCuQcgb6;XrOa@te#N%(M8YBN{JM?%nEKQLZ1o$ihwmQ!Pe zcRD$oFC9hf3VeX&Qc%gEWb#H~?zaN{UEH}D$7l;7MiID}B0{3IM8q)_b@tS>kbWrT zhAu1|89)9jEx+b|W_w47+<*n&Wlqh`%7vbk!dhYHO zK5f5u$EUZ2v?}#HUBLa(^lb0<=h=xzmzTK1vv;Ja?sS(lW8H2!foHC`8kO}YWxph%Pg}7WQWN$s zWoN`uUXRJ?d~d-N;q{`;^m;-~`?y5O$ek+ke>gth+~i$ODzxnD5OFFEeo9;+HwcQO zSCoCx)EqJG_W~6rD?I<=hr?CO07)~VwJUPJ`_WA5SFC){&&7v^-ukeBV)HIF0aaTC zbFFStz5j4j@y*O_*Xa|p(lzMKLm-xohpyC4u|m#8F#@xw%`brL5qzWi*}qm^;Fn}E zL$|t#n^7E-^0GkM66aDGP&9Kwh3yhCx=U&U+=104g076Nz{a6UOUHH3LN_q?s>0_5 zi8ViM-BHDEO(;|*H+CgV$?5hMtQW;8%udIW2Q*Dqg>zF1Y>D%dVVL@Ph06>swE{m` z-8EqKr>8Cus8u)imjv-!ito};O6V2h<$?oM@IV^-paXIYeY+=h+^8;z`SUD&mp{A2 zKfO!eG#(xZ?j95#9t`duf_jV`J37K8B`L*##U7P6Cg3*9_+nX$iN1^z=S>aiIHKRsWVV=%5Cyf(Re6+p zC)^K7TDnV>q#Q=kM{7PPG8J}+l084oUZ|(2ie-9?6?(@Yh-Tl0LSIFOVo0{%;P0Zg z3J3zd>1;Z^>6Tyn3XlISJ3hsv(Cq!A-8+VeGxt7h`7SyiPq12k_!RY-f7$En@Xz^c zU*rNatWYo8{L3)c*J0hQIk2HVP?5n>zzVC%0xQJGQ-3f~oe@UN1mF@4HxIAEq=aTp zOL{l+ZGm0pBVL95un31A+7O&tFvzf6YkbNO4T66FG{_*AMG)#xImmz@)<0)}_3#)~ z@N7s^t9J+q|8pVMuR?PPL~E7%cTr!3XT4r5wymIsg}1C`U&Zwzvj5iVO)b#RxL}R! z|E!qvU#3@EppOjw?~c{|oWca~6!lOVW{`cs5G_1UKZ8gLSM=**`hB+gy4hcEfigK) zlos`*lk(c~NYJvFw1YggZB*+M`T=7p2>*dWTq0x#x8%;K!{@VmB& zPSjIK)|P5RZOc)yn2wVW)sKvH0 z5ka)ROLm%8zO`4c?O$f7pKJPfC(h%Xj3V%nABTz3Y#YL}dlJ3{TN`W2Hq)>x>RUk5 z8q4O4>{g?yAu;cYhg?-xKicg6gyMBb=E{I^@5^7c0hl5WY4aVGZ~5YP-zGG@0TBFr z`L@T!CcrSBRgoU<7q|7J-bV1%-)?%f1h#v`!7)syoaW0QTi5_+mDR403}g59Ts@uc zj~@fa@4g0lpZ{$UT(9tmKa@@QN}>G%#GG`3UcjMU0C);cWEiQmc}e{YIEOv3#H8r% z(;#7Oih3r3D@q+8#9OUsVlgVbwIMWByn3Lt?VU+-dw!0ufDSa(H&#|FfFuWH@9k8{ zwB(SHqJfT%lCu^ zhjRyq+XJH>bp1q|Ij*?TS!n|?g7)MFhu>QFNO)3;?}QFQMqpzw_A8HpSnjZ_Xn4@zkuVNi;s)&!%LbHkyQ zYl}ud!J)?SN9UR_ipw)O;77>{M8)RTPa^=NyL}&6ySt|H0vLI?SMCTm+$;C_Cyw3S zNCfxsz43t?A(4s57zyrZsM%WFm}ZKsTTa8{d}3cIv!0AEAtgkLCWD@p&Pe(A2j~vw zSQR1Nht7f0{*=SVdLwG<1`2ZHc2P&_zHaMxN;&PFQ@4`fl#8h=v>~W^wpZMw$@=KY z`xuq>V3#i7Jo@!_=nPkY1U#7bWxw9^-*)9K1LPrne%d^4dD*XC^Vk;b{AvGDM$m0k zSRc3yhn1Ls!jdAJK+p#0Nb=q(7X6+gxrj)i(6hIB4~3M5F=CAO=f)6P+!n4q!b1c@ zg0FFoK-Y_lo3a_gE4J4&0BAyH#uDY=4gJ6foyY>_)Q)4h{BYs6uxL~8h>I4Eg`6XD z?@vRLKPK{ufn9_x4lGYQMH%8^vAp(EjH7i-^?TEHCDqC>&-O6QN_fx z&F1wboW0@iN)V1SzTWa3bKK6x5%TF_|CCS93yNBAlCdJ}kzVnpD+mm{e%RSVBFt5$ z%WzGmIVRnTS;>J zzpMdgXIwIc*{PZenPiHfW1oUy_v@|ZsFcCU$l-UqKplmfVyyhHeYhCgq>GlNl=ad} z#J893u(<26t0&Qvof}~WUN&t}y|*!K=bDFWrU*9Vb78Tw+TO;pTQU6W;`j>HnToNR zS;0QOXu{KqN64=HH8<5oG1Y~9RWv>kc6`aZcL{$^0I$r;B7FlbU8nkh#isOtWof#N zYPubKtT?qWJ;%dhb^~i>C-*>ke3P@p^Rp0En!eEE()MxqfM0sl&wfmCMKIgYY37OE zX`f#X;^c&4?KN)UgzMXgTHfSS+%eXT;{8C=`GWl?-SedZ(t*$<0YAb6V*F0>`}RLw zpqxvkK#Qmmr2wH5lP2-90s(#PLLuQ$g#uKR0Au1`hBivC;fP5ppBlZgnZud@`o@BF zj>lS~W#f!WOw`f&6f;Pl%paX_!^2FF@c2ymF<@ul#`&5d%Yap&&c2aoQ1GTk{@o;h z)+zrOyLetu^2dyFa3^4&z!EVisAFV5n2ji6q*gfbp_4>R&QGY zM}{R8Yn0wnJ*}IVm?hgOu;tgGcLhw0A?|qP@JpR#^2eR5S6B|u z#~EyzAIg^qHCG`hfP!h}+H1|*6Xff*#t$n9!ri$!^M6GPjalEtCb}NUvaF(Fux2?H zKi@C5KUFETIL+?>ZO&?E0vTV?FX0gG?~o52^N2kzpBd12({Itu>FnTo+UFS?=3I<& zdciztmDfHWZciC*2cJ#$ZM~y811QI)(>-?K5cF9Bb{3!djRRF zhG!s4#9bqIpe5_x`>tsMpcHhcwTqVmX;~G`=%yOMZ{wgeo7VeKFy`_b(G@fhOJ#^bNrSZmu#x@_T`6&% z!LW}6gJk_hHSm@#fQ1m=)&Ge@ticn{gn&=8&)m&{)1SI27N-1tVMrI9{5AujJv2Wz zx#^0JBpOUWhVf_`$Zy38FA!V;9O$V$Hut%Z>@wE?YHX6`*H#-ZbN_tQYojr~nGiWB&otT_+EiD}Va2D}S`p4q_&WhfT z0pZk>-_P*UASllA_lKzBgGOmycQUt}#r;1a401+QW^-81Uu`KLy{Lv#><-Oek_A#p z`j3=On8hXKCG#N%0KhP!|h!Abn2aTrIun=u!Qi4+v&Q zpQ~3b`5G}5iXFcE9dcIL?%r5mrn5pAc@@wdZfoECY6Pb`uCi9GdhdZRG%Z(P#_G}y2l68{L&kQsiQudsR6lXGIPbD z+&j6iR>=}4;VR3-2lCvKCFCoB49mE=Dtzdoqm%MmLRbeyBM*!zoBPg=0A4zD^5 zS3T)a+u^GfkZiByuN7nd)?Q#ODx7B^p(l)Lh`0j+dYzVQG9mD-+E6Lz$ zp)sZMRF3V)sT;u>aQ-*z;SHMjN}!GX-^H{GW)Z!|(W~U_70f+BIC9ZX>z5%13lQ7~ zN?5h?Me&#qUinDqC?)8H!ph;>0*6R1IJ8YJIw~sE0viz`u%o`?Tb3GK_IdizI#Z=W zS^4vH42{KpAddQ^W3Q#IHG^`~Yt69|@|PNuZdO$M;OY3R*-H*T5lsD4RE^$VsH+-* z2kitJ;Rcb_w2$SqEp$i{tP zlGuw;zYj@t`jTIrb8u`HvpUgMOqke*4uiiYUw)UY*;26!uhtAO-#sY7*Fb_(&;B!8 zagMfGAQ#jE#>QDffL|i{I_BM5Dfr!`OsN0h;EN&P&Py^{q*0dbKY-ZI)eG#td%7APT$JgYJ1|*{AIh+{#0Ty;f;Qw z4ZCz&Sex-eob_j%_3|5-3ZUy!iP9?JCM+Fq`R!Ra4&*u;h?m6gOU=J&H1VO`HXqYg z0n(0CtlPNh#9~2{1p$+ON%LL`j!F}Y?smRe(%$A|pmwt|ceh;HfwNyq00 zSn>Ny`*>ecKjA(poMtMEbl#;LjqY`C(Krl$HFT+UaG%1@pzOE(AM=*fcAeHTfAjM3 zqwU=Te0%fW0M-mV7k&H2d+c9H58r1Gb^?C5pu5ZnXY5}o8xtri@!~XD z2WIkt&GKU)MpAgU&E~Orl^cjlxXLrUt`6QZdpdT3Z;OO)aEeKP@!GX@-SYkpx_0Fc<&WNtI*vNqvV<-P4(6YT31e(U#xk1V$cNI@ zm$A;;91kL2?QVBR`B!~9jbec^dZ#mGHmYpQt>XkkWj2~k;(RG^NdT$0iKb)c#}jfq zon$tek1N3hy_xrm96(qIhl;7KEe7Ly!}_=jwUKT#R7^t9wfI31yNx*PTw99R7OrGn z5S&UK=gf4tN%Ee_K~N0Utg9Qjw0+_BQ4@IG>~sToEqwX;d=$Q33C%H31>&Vtc(X|b zj*iB|3p$23c1#bG_0?K@V1U#NBwSC}9rc&i7EMUn{3UCNq?j8NOrsZ|LGdCl2xs6t zXbYlXUu~i5Q;j^x4X!v1J}$65@pA_v7^Wi(T6U}cz_$u0I`4V<0jD5yEUMfZt?I#h zfaH8uz)s5UNnLBd?yz=()Y?jjnEqWFBMKSlE*n3(G5}eC!+7dmX$Ha&5FAel$g|ER zh7$sPukl1p#y+NVma5zu_8fDW>a3Set(7jZX(uRV*yxy_8yc4or5J6(rBmk?t5HEs2tkLa1&_@ood03bW#dCK^L0XzPKua>DSa0rlqH&$3~_`t4h;Kcb}`v zyOcUCX4M_7dldn)8NLpZdMh7YQRod;%}l;;P7|6aA=#x-nA5r6YkQT(b#;7zI(ZK5 z0q#_)vZclx)k%!|1W{r{s~AZ}UI2YR{f46x=PG>NZw?sg&@z<4UXw7(;LzKQFgh(| zi5fME^O2*S8q$S_UXgA_-rlYE_3d8*WMCr>k?WZZR-WHRM%GM?Sa6<$FgxO6JtF=a z)qR;lLM;0Pbo;--t*Pc{X(}DsB>!riA+EtaR+6g=?383-lhN`y;WBFZdxbps*LHmc*Wdi5ht8i{N+b(E;vuD1|XVeANW6Ly(V1l?WmuB|z_JTQWec_XXq=3c{o>MWaZTHzR?9G(c?? z9~E8|5?nP9+B8pNi$>^-38$Pzp@y&=Y<f_ zsi5F2zToUXkkyLV7$(YG|6gCTufCAQb!A@(l`)aG7P)a_LlCug_D6sM{RxoDUoPAj zd1#X6yc-7O^QWZ~d1Wb%wg&DuaCuAPok{e0s1}pp37ubxF|0h9w7}dykCfh5bT@rY z(+#ZDmKe^KjHb8J-eyHMJ(m!bGe(2SgO-Sl+z<|$;Q?uQj1^gcts}$$OIBO>=o#Nt zHj+qK+~Kt0>@AejR7?T15pEu&A-x|u>{(5_NMqEm=uk=z@&Ap1w(4ZY#!=xpd#lAV-;G+zH!L77s0Y zZbBQRL~)|>$s&)ePTZ3i{D77^5QaW#^@Z|^7Z{q zs}obqdYnaFspXQ?Ib_0CZeB)RSYyPcR(t6N5f^o(OCBKJV((Eub8SOrSNUR^O2z)q z+pSg?CP0bv-cQ&Sc_Apko7Yw7dZEDXjx)CIKCHt>A}apBCLGeQl=p23pS1u|cFHVn zdzMhJcZENuXKXt!_2Tbr{1Zox9Dc;J4lZN=nRvwH2@IwscekSQYb>zr&%_1CkPsr# z-Kfb0Y2Y&CO70iLmd1sZBPsuUM9f>rvQRb(1L8eD=+@+TBb-4s7E1Q2Nt7~SNyX$9 zNJ^mkrIBTHlm!7Vv|ExS<0yHv+Yg`U|IdVnpZm!Ecm5Hw5oU8>8zjuOd_1wVgh^y@ zE-02W4j+9TsXH2<*%s#4feBH_T-eM$m-NB^sMM zN@5*XZ6}%Y*v=Z>){o@S|M1f|*91;q{=DybV6$Cp4rD-f?#qYc zX$xD|WerOt3^7A%c|4a>Y!L8}04`wi4t^W5K8giF5* z)SzDk$rgK*j;Kp|V8dsR7+Fi)YG$o)@O%cLeQXCiuQXRwRA5t3XWdo1e>;tEIClIWP7s9{sFnxd z;Uq+8I^oZY`r!y>X?a*@{7jGk5P~g13KHHCl^>4i@L6VP(7jI`0cW>QO}(MkKMsGf ziP8DRP>vc|3&r~OyfCEXRC_3Dcz{D1$cgV8*eo|!Ixrf&qca@w05Ru6Wq_#Z!#}>E zwQKu~O9mzg3<>rx$22DZxduFD-BUUDKj`Y&ELG1>spiN&qMKxpyEsV2GED*DA?Lwb z8Y?EZ+LAWHEhQ#US`XP^!(|M$q#K~@98H7}Oe$K8N`-6o0$$~rEQ6>_vt=lqqkFwi z9j%LseCzm6q5iSQV@jIxG%MobK@VS;&okn@9m7u`CQ2PFLhuJZe6JyX-A^u09*GFo z04yV2H>vR>YL2;q9E*95UGi|`ShLjz&c-$B5Pzs?90k^VHPCa!6Ehrn*jr}`opT%o zc90(%MP#|E$}Q^h76WY&sDl;gD-VoZ3>8>cr<)oCFMyb4bC-%} zfc!<(ybZS*ks$%j6T+@DBI!36i(>g}WCv;vK8sS~J8Q5@P>&qc7kUOGFgj4QR+c^n zP5ov&zB3a-y*6De;!7|BS?IcJ$VM^ZW}gMG;kAvO_PZwi$P(Ib%AYb3<5e7V4*W-d ztXvFhLlurrBF=Z}Ii*K^x4D%$)&(GcDtL-J+9t3q4p9{f7cJ=)s5*Xp{IDge=`fCD z3bX;1um%}o_2L5c3S^iM7@CAkYGkn>KZ`PF}tG z7&$;vTvRatzOq~9latr-QZJe?>W<2?@hFB4xr^>ISM#A`T0xCK>`yT5cOSzqxvaFr=PwmI`o8SC(MEV zC!@~fk&V&w`jqeLBcLBr_=PNhubC+9q-p$d?yWgN;cTje&Z%>CAT;ZV9es2kp~0~~ zb^XjMS#!Dn$CqPO)~CaT#T5lUicTA*^Q?fuZW=kdIA>ujGn z30S(xGykx|ofY~y@OS%h0b-xd7!Wx^k#+aD?-+|*)a*na=@RsLCP`}kGA)VgU?WYC zOlv{f(?$H2`{$H*j}W=p^vQbMIO1!2f1?6ut?bRndFVHn;7O;7+;O_XE>fflc~U7xd@G_ zqH_t|9t9kdOyj-MT(7aYd{Uto3KD0=voDQmH>cn71Irk@5ki`wH_>pTGLkp?Y#|fV z;X!@)1?+4A7@!E*0mvX|j82g>dbj-@!|i!SF(47S?o>{HIf)l}jdAey8QN4*IC=$} zCQ~U>u=WYaM(Ebwd-Q;!3s|Rs= zL7aCQ!h?>bagV!-(&!5zW~SGjqeeg%Ou5LNnR63WNf4|t}Ma}*2G12*!RNmYzX*@Uos)+pt|rFy6jv3GGIbETKk_7 z5XmRkl?5NY-{p%3&qahIdl@^vsrZSbhm7SHd9%7Hqm~9^I+isDJUVj%8HsgQ5kaSA zDB9c;XqS}6SgD*!FO}3)9O9t7lz@j0Z6k6Uq$c{ThmQRFOdO2%l4N9ieeNOnw$4WK zB+_!&xJnY#oa1{B+}{{ZsKq$9SLN#RXm-==UdE6;L5JqKsCMl-77Duzo65@P1Zo5> zSoD7M8$IIE99_?9qAByG%lFHNsENpq;7vC$>Guam8(J2O^pB9H^@apIA@(^Tmc-s8 zRrdv(D`SN_wHAnz-3lXDph{J}u6Ku(!Mq^0d!nT%iKHUDc*X;z+qJ8_OErnL0rjW9 z8CrlEyqt))yME8oe>)T2a#k#s+*1egGlk5iUqY|xF7c=R+WCuam$?r(J$-+^b5p%OIx9<-3?mti3Ie)H zxXU#0QC_zmv3M<~7Mw<;l$HBhhq79@8aesEizAX@RR%JMt2lD-AHkiV+L^F~_+>B~ z5LS_X7B#eO`iFf{3OQknJ;;V=ZXr9Q>3;F^?X-UnP;C1d3sCq-V2C&bhYOR-4!36R zM$yeFBp2lO${@HcDbj2Jqks*TUjhS#u=vd*zJY1GUA+?)kxbYH#3%N|W=FUyl^1ij zO{}j}e^aex%zsr}6iC{8WjxlW++c*hpO*@l7hJ3gfxy;gXSva8QPw1}$AD8Q4oHu2 zvS4da5IJk8j|u(0jOD&@^Cl<6?B5YbGp-*ooq6KHADY zR%)!m&R6}bq^I=M$o*Cov5N=u#n-eOexK8ss=RAC5yx<<$kmb98Jb5j0`oJXRxUSgBYNBTIrXtwvbd-}&J}F&1rQLV zEX$^W6k1v`N3z6$LZ@8dh7{tYHV{RcSm1#S20=9d@2k*dgNMj&54PxAqwb@U5LlYs!|STG$_ao47@=qT7@$oLb0IZKZ%Ci8RnMOIlnr8mn<`Cu{61<6&0na+6f7aGO>Vo-b z$*oSDIK`=@%$)ert!L)5`GMQLXO=#D>(8)N6?cGS(bw#ln-z4cZKIB>bT)}%&fQg- zSoRmi+2jsV>Yq3Kl`eZGpUsJz*j6cPip8}`H@ue(04ZA^sC-M$JNowDU3m?F&@y2k zz^qzs$8%omd-GBo?92CMvt${3yD;M+%v4*ZsYYLSufef^?ef$}B31KF>6+XXl5ZwS zq<_3a$(t(8f4;4y@mK)eDEVL22Qp>VZ}~-FeUwpBI!BM>K+hZut-10f)5PHEXH$55 z)scgtua#?FtCkI$pXbRIJ{4(a`Aaqz4xhb=->c_IQGWP_ z4kamrrr^%D3YifYs;k2jGX$;j_WAP($uCh6EO@kCZ*WNM4EEPtcNnEuEmd^YaBaLo zK!jB{l4OD!x}ydco`z=Nc1b*wL(-69a%wcm$e_93k9ZVfV`V>3P?KFSH&|JsHu?*r zzEO+fopRu;` z_(@6VAY>=kvT+_hkVb30R*r6kP@pD#B-9?hxC5KQwCW~uggNEbTDF!_yJHC7YA~^l zcfS0yvk}80kTY;sPVt}L(=t3#-yg4`_ImW{dQ z(cuqU2N?Au2lNC0+r5dU;B$^I#)|F{_5WDk5-R_wn80No)m|nt|Ue#Hf zT+zZmJS*fkm6zrinG$e?alAIK=FKW1L<2)7BLgFcs{%glvQXP9n&;(wLx^qM2B4R8 zlu`UycJ{&tFx>5;aPvX$Z2Z&%7#QMj+y4odsylM_+VOKZ6sPpHR%gyF$n2oJDmCrr zHSXsQrcjQpF16(n>$Wu7tl~G@+MSAe zGH1K>9xSXu@=O3i8h|5&&WT|M1k^P1A6_{p163N0R96j@z*_t<-?R@^L9@fNbX)OT z-S}tQ_imoc5MYY)<88X@HLrjCq3J=8{QKncQacZ@IiXDPEiV3jI+k}O{(WgllE)%; z5uXPN^_F1NduuWKG9+Zz5VIIDZ^c+)*I;AhcE6V82T@w-oR53C=aJ3?T)&=?KA?%o zjfpA`PGrkR)vmNy6!aT^7ikCGUh}c>bDDJZC;}w@L~kl5F9ACs+-i2;jZ|Zq8@sL1 zOt}}7VG{#VsS*e`rac(|0|d?wyrXr7Rf(nZHp_uSx~8VkeM2To=2A`0n3)b7M*^G0 zBVl&kDUhyYjQ9A*{6LV)6?4n!2VE~6s^cw@(?Id_ z1s`pn!1|USUiMPeneEHpf#eVDXE>*Jt`X@CBL3eHY}~T=$j!+DKiBTz<(daz&7&9I zH~L?vS!cYB@~K4N6Gq9?J=46%Z#`vq{Sv9gVMzN@Sx!IZuxuTkC}ncjmp-f=sYt3C zB=ZpcIy4Z5&}eo^>Q#!8e7FSmYBcj4eN5-Tt#O`;l)x)$@H*=-hVT9p!jVKt3{}|1 zqZY$_^dNM!AQvkjg+_`yPV6QyFN_PIKoDF;h*z`@{Z}2q-b}a$lTv_@fwh{m2&LD$ zoy>VuJDSCT)kVXls)zKEd*cF#e|5r`Aco%hj|gRZmNkG%^F*!}9&?uVxDUE~T_hE|)0W;+cb zqS$yWF4{$uUS9c#tq-vL9R=M?Qq%Bvnu(A0FXgg0}U$ z+X6^`U2VoVcHO}KYa%r7S_h$+gQ67#v%)d9T(NFV1D&UUL=DS_vAE}QOt^AFr|)^J z3cK$`)b;%M9lp(j4kBTAe&)#edSmaaz9KR8gcB;_k!&6<1UlZ(jJ>t65)#2#f8kU9 z3V8!*ic zPLWC)C)|{Fl6SBOY9oYOpoRibz<@LaG3yQhp^S*BY(lM5g9Xl=sjKwGFHxtv^Jz=$ ztJT5L{@{_XzCtgbw5{`2h8tst!Ma&!o)Wt7dh(Ayga!D`GI(7&3eqw#N5{$Uq#A5F z`MnwW?Q|?*>naIh@)IXsE#^Z0LT*Jnl_Sc28cArLwA4o3-_SHi7OTYt+YH89!zi!! zC~z1kwW?ipq89;C%PVY*cC~hMTew3Ba2qvb@=n9eNLw99DNYJbC%8?NPpea`{pJvO zLJ8|4*DvqF%b@(7Q)N(H0B5powg~PK)I1W{^k5}pM=$c=#wjj7vWrM2c5wL1AL-J9 zPijnEwA39kn0{&J^mtY91p+$`dw;LSVAxQVi+y6Ci$>(`YTl4_qKd9>vQXz_akz&1 z@lAYgOQOuLx$C2(s|Rx87;a$ zQCST*95+xf&0ejKg_Cm1tpyGN{}SgALy-!QBtaC38!ddT-Y@Xm@OK1lL?)g)y-f<| zt|v6aNeV}z4r9znB8VYNE*?{^xA)WaG%icQT&0yhnY8)iDwk*Y9%Q@{2leXW%%w7a zO&rrIxY`(m&?kxh+rR&Zv3Cm2B;3MAW81cE+jcUs&53Q>wr$%^CdtIf#I}v^&)&Pv zf9l+w>WjYZuIgUzT8|d5qApol7uyuwPbRAi&Kp{z{?O>$VXHXCXnREEd4?c?2`X$C z7qE#BB9n;#$o$DjAWOkwUM*xX)ks?E@dOy#B(OI*BwTaE1c+j~$s})X=8xl0plpIB z=0NYpZm|INhpKp?4x}B=W0MU)df)t@LrQ2GU2~e)gkHyBDNN)icS~>I>hTN zqGVYPmBbn*^@b=Iks7$(&1y1`wjI7S&oOQ}gvM(PRb>M~$HFX%D)PgvNCHX)FLCJ^ zX(@Hub$b8#I1B-4gKj zskZe$_8i|0&(K&`eH+L{Nnpk@;OMb+S-NPkcJ<7k7Ujx8A~61k-wL8!oP$%a_R2^i zXDhG;0F7N#&kdw6stX2BLlI*RAa{!IiUW9wkMYMp?$uvFr!+GT9^7ki~ z6n|@hxPgHNSrYwK8uF23P}zSV4gZf&dH0I!6XrjC6@=v7y1?SYaPI5>U-g3k*l_{1_NTfP4#beLPqO-sa z&@BPsR7}u%C_)M=@PyQluWX}mTplb(K>~DWe(2m7Pthb2KGq}f}tYnF0DDR-)ayC8{JT@+o+j z*v!*j6Qa5A@_7ZLUmd>vK>tE5SAVh~q_GFgFx^#!>jDt2Ja|oOKH#}S{`dv^qj-T+ zM)acJEYH-c=xQ8nKRYVUg8d#*15%s`<-3rV16C^%m@v7cW%XMU9Yy$A-VkX$sM#2f zbRpAYk48~%3x*AHjF!n6ph%2KPt7FEa7U@SOEq-hl5?L6mS7JAr=IPkbf^@cY?~i2 z@~`&ZPIM2c=X69>fmOgR_#=YYg2IGZE@C0#OE4g0nGvAzckf`H;MF;E8l~pKzRoCiV&@mY5%uN*Yu;BT_;rJbxx;wsE?zjBG7lX0O~2x3jCqW^;RCWPfs<)& zjJ*B@?=dt7i@8Ej+~-s}hJ&$BR2^LDIEcAgTEfc|GISSZdtv+i1j6hhVDF}2u6IeO z(N*k1jeumEo+aU&wjsC-vH_JH6US>ZxH!F~sn;D5S!D8*Xv1Evbby+H;}cL~Q?|pR zFRHKH1r>7(gNEH0Gq4h&8F+*KrF2)(T9t8&{ris~P^g(M7e$2)0gnaYQ%o1OJ9!iZ z%M`}ytl#yFa7IXd%oma+&XQ!1L6&0Hace{uyaZnq`xFmu`Q7oaHCo!HV6mn_? ziiXY90{b**shx&U2-nA3%TH5q_zXVv)03uUen+sdUcj%}05PWfI1+sTA6ODXpmn3Ezj_0|Ds)@3_uJo4|agC zIGGFo+S_&E`YiKza=ZV$#GT(61A2`SeV6w?YHJ!{KSYl4{k)tMOpG>F!ogDA4=A7H z_Jg4?UavGL%kjS4f6W-`-UfT96ui%zX`f4+_NH(|ky|9st4NtY!1WL5g0>I(|0@77 zMgQfv>}V#!AjRQxA0|~hlOy6D1mxD82QU9f?{NYmLN5kB1|c{F1m#I(6+?XxKw=rl z+-T=SIT8`Ol^snljVtPaL=S>000#KSJL`!nPS+7U#IC z16U2mnzzEE1N1fmFTm1;^`mh}{gcaWCLblFFm!;FltU$ zp3ok-bWH?W555b7%Xp$w{E^+!)VyS6TDGyv`^&yxeLmOi`is!uQU6ij#Q?2ek7`Cf zDx^!bs5!zPG!0cRlmNlH8>rFcJTru3=+N{ZLf${a)h?i4)aD-hL(oNU@#X+y}$ z37+$6^x^j{HI=7VX-ndHC9Mdjo~<^}G#(y6&K0WgEo5{M6&@UlsqLf_`Ng@AjK?;0 zCX6MqsW}U(b{J?{wCgLG;d0DPDrJ+**AQ|^Ifs-ot8L1**7R=ho83yGem0_W)w#f} zHID?@0e4Xb3+J#j+P55`|E`Q{JwJZYgOmJK+J}23P+AuP#&$l0;bl5PqJ74j0w+}W z%<UD56dMuom~yYFg^SaQtjJl^Uon~+_ogHqHAgE7>4?_%4jo=|(a+NsqUDet{G z=>97cJTtu_Y08zyHubxP$byo7!lRaB*>Xnp_nO;&(z8K9-E!mi@w$I-a`T?n*f+1Q z-^%Lq4o4o$hvMwrwWk07C-$NZBngd4np=8t>h#2$T(>-7%>auTU$w^Q4OeMC&t!T9x zA;Bv?KCmrTyL?A6US?R=0LdB-E*uw>t_08<&JC_cC1Feyd?=5l0;#Dh(n0euNjlJO zlcJi4@NnbG+b5gqgQ`oo+Isx?`-tW9fE7WA+SmBv?PfRHKG_OJg$EXL;Alu?*q#s|DZuSf2tN+BGLe9`T zK4_7)7204EB6W3%0aa|elQBBEw}?)V%<sM`9J9$K~lD**P`gOdYf|1V*X_7lZ`pFPN-1MDq{3JvU>}*AXbrq}DS(Z)zg`%~2?Z*sO5T{`fz^fWv54>P(I z+MWB}FAfY2^`M^=Q9H9@!(Xu???ji#54rGN3j5}~3k9LlQzqK637Vph|02BYD_Gkn z;SdoKt$I?jO^ue%Pq1$L{Oo+ceCOT2E!zi~*X&eQg^GJ5+uSwKY&FTdvofNr5cs>X z>@?}5aL>~#D2@{ms7^BX4HB{!kM8)DbwNKBfIf3G62C+|r2iHn8So?U#@;nuTtK|- zDH!|2EiGH=LLH#q+XxB?$R|j6#@-jHuClJyWG7WswT!*i8h>8!*$4Ce4kkIQ@GKOB_ObNpBj+*JyIz#q!n>VZ5Oqab}pKwmwr%slR;d%HhNTh4Oy^%aLA5|WWo^Rw_W(7N(z37hW+DS& z@`=@=)*$>Y=vy+R7e1wdW=1cynrcSpNN^b{6TH6KT554W5Lk z8o!Mckw+#_h3A{7jAxW$>)=%|12ZmV))51!3uK0)!!Sp2K7WKIw%nS^X6!k(=n5zm z$S}jB?gGrt`BlY&%}rl(FdxbP=|Jl|6h-y0_Spu_pkJwRpcLtAkHrvbbJOnKc z)ZJXjExUWbzTEUIRjXSAe<`S;6er4&X5P00BIV+0;tOrH?ep1HBgfi~jd0;98h168 z{Oa@jFk61OriC(!Huu!B~bFv1;J;=pKS+daK?}F|`5M1`av?P`uMSlb9ug#zkb9r{4S5kOE$IO@DKE$47v*))qpZ zt?|C{4=C@C_}6MzviAsofgCkLp5gIri&qa24cW^()E0eI_ZS;pC3U4!q)*_BM^HKk zz#MUGah!w|XPPb%JG9S?Y8cDawPX_e4z}=?ye7}xKaa2dj9G1<_ zqIWghYQmb$hca_Se}75bwFZaOhG!<*kiV1HSbar|*A(y`pvkI(>^x;Bgt7fI`n8K8 z3k=a5v#UpHNsNiwLZvxjD)sO+vp$Nl)43)#LsekjzBPrzy%LLe1N|c{k*R#{7r0-L z+(B>tUTMD)S1PF@h^4|6Px)ocFj6Cl3iOS_3N+#2YZ;s=1j$@%c)9uf&P=-Qzy4Lw z={O@a>L}0<?y3JNpY9~U!M-G`Bf|I1QQMlAMAEe|?Dr06Z~4c?nO?y@OpES7 z{iw?nHx*G*?9xdQcgm392u|7EJCX;-)wiv$Td*vE>1%c!G^%@7}e-QJrZ?2 zlE})Yz~M?IDPh8Fd6|iaU_t|YrZ9uqIAJQ}q|lq|$xK!*bOP^9C>#W<_Bq7q^{r>G zi>yB~I+Ir58SYe7*-JLu&Wof9;?D-6PyW67$crhd0Yy`*-MR{iVOLh&{)|ec{NFg_ z_Z`h>hc&djTy{m0@=hv*GMMMrefCae!0hZEVWFASFvK>aTYm^o1}#SS9iE>p*o|;# z5MgNq_4?n=ietzO9SV8;`+!7_Wo-;?A(&u_XGQpYmnhSts2#Z^$ACh;BZMXl2g;$= z5*&Z?=m~~6na8SC+vXV8I+3EW9#22;6s7DQCjLY>4%oLdb!YY)&}hSx*sN{1$tjk* za>zZ{P!DA`TRj+-CE&#}goiV*ApJ{ma*z(0TrE&GXH*`Gq43(2LT&G%+6W;h?Wlxi ziNuhtvs}lUImA>uC@f1LgXatf=e&XTKT$wJt-izI$?Mf52oBoEvCINy3p{`eUxdua z>Yo+c38`&BOIGW8l=0T+Tp0C{tW*3M*aaSctp%H{SXTfPTZpr^jnK$=aH0ClL;?9|(c z*MCGNq8hFEL^Rrtvl2xpQEX`V7-+s?2z#els4$eLb{%((Kp5y61%nmXhG_Y67Dc^X zkmwVd*HNMSxye2x8Z_%eNL$d;dWZV=mH5kDZZj(UhEdGpR%3J!wWlgp+*r`(YU)H4 z)oCwmC$#VoWEG_U++e@TE`XRtoNqI&*lg|iz5V`5;wJj&p6Bp)6awvjghB|FEeNj^ z#|~yi8VtWBiu^>hdnU3ZxnL9oFLiv2GfN5xqcOaUYD)qz*xqkGT&08M1V|il|O!V4{@aSvf_CF0`$fTX_fYQ z(>e;wh56pxmkt^PZpD!lEOuoR3<0oGhzk{PT(u^S?3DgeBVZO#-@{2O5li^ zCoo*JSQG4Ll#J;&5r%iQrVBKv0PkXav&z5TT99R=Z_iFTnvqW0nATwoDSxVCz<$G$ z(fqDP+rp(fr~!667}gWD0JXf?eB#J!C<5*NkNArx=E!6cx;mFuB=!45SH#!>PXlk% zyvQ}hq*)~ZTIL7{LQ^1#8Ihxtwyc5zjM{+ZI09lldCAZoGAt@xzZtkPg~DzX4}+!B z2G)NR;ke(hPYEJgO*&R=R+t2vRG%j;l?nFZ=Lc&sX413OIdw{NiJG2fyzHiDfM>l<+cEvZr=%3O>VR#ZC*N+6W4Ol%2S ztW;JQGunuj6=GAkZhv&c7K*l1+GL&BI8!F+$3#>*ZG*SSr4eT$j1nFjlwn=W5n>jE zOYU)0TaO1{U8Lni)&^~Mw%mm%lvXS?yk<0*;lyb$r^_0#r{$APr>qIOCIMV?DltQY zNbI&6>Df+|`C@#Wh3sSWg0Zm{1?G$OYYymve{O7IV}LvZIFWS~uROznK|8b;k){Gs zlyz12*T%~?$cJ|16S}YhDsp)Q4nDnyp~u6I1P8Dx>GDh)2%EdBh`z+N2q!9?i|XzA zq%dWG%Ut$HBloc%^gkl%21qEzQB03eZde&&_aHs8IYx1cMZ^DuX8WK@@0WY9q|$FP z2d7GatANc?yHPgvBlL{N-Hf;}iJ3VGCwG)7nz88q(DBIAMesejXoQ|BBe+F^)uvj% zR%RXF7OGr$CI0O#$Nz%%xcE|n|6M7*W$R5N8^uwA0J|1--RgJrM`d-C5y+jrYLTeJ z)ufuKIzL#C`a9U>;iha&VtxAI$TiF+csU3+jMLEYC)O7~3QnPbH`o({E=>9`Ys8(q zXj1j6d?)lE1-peSd6=`wNBt)bdHGdzxg^q;CJ`L`t|P!3i2icUOv8N0Odaj8s!e$pFjDC zAH#!ZzhC76rXFr90EowSW}v_7um2ZEtMUIIj#kkVD1J_&_~87>#x+RckLVmP$CUqG zoF|?0AHb|4n#)*PU&b%`rg1ad@cR%xajKa+^coEK(v2TC>s06V54bRaH_G=!c}8HP zQl=UUc2oiQ*ht%C?mU9+<+d&#^{G6`Y@{^axI1$`zGC8)%Pe(C4!(2Lk0H6y8_$Q> zD{Cfhh>Oi3u6(}7^ZpXc1(f;@@h;0r{rms%L3w%qUmuhg;2>VK#Xr4(gXry_>pFVB9X8Z4A%NnO z-Y{&Z$m)vt7Y>)%R53m#%7_pm-W0PC=JooX3ouNbQ+# z-Dtw%8LhVX!Uk&Wp*{Z?o}!!R=WT*D@)~t^VRx6@<$wFp2pXSk5XRr}Te|ANaFfW( zv=L5?H$fD)>!?6`CTVaMoGjOoSM2Tq&lD>Ec%Fo?c6ZnLXs6=}eB+?edot=fC*aa4Tg4K{h-+px&)E1P-)N$xaIg&0|O@Kj&UbNENK!2WjU0I)qw3-lIR;y?*81Mombt*OVrn4<*jHHn(`YRd{#*#-|e`ZW2ohbjm zXo5g`KsV*_Q~IUEr6`5?Gm+8y?hO}(e^wPiX>a-p+qr6?3t;2;XUJ4YJp}Pv)LvXF z|Lix%D+DkrWV7AWN~a!y`^y&b$``e5eJXpF7G`p1p?38^&;4^4lePboy6D6`9necH z2i@H{4hM_J{RChyn@2=H@AR`K4_BAZj_^+`VDQ;A@L zVd z22DtG3`rw}Wr-EjMCd%9*UDoZjUOk_c;~wvnY+6Sp}LOy{(z|9&G_#q{ZYavWn~_e@^}QH?9o=Ul%}=eeQ;Q`Am{AJQv>qC2~{dy-$B^DY&?g zTCAR_QU7nH3zmggj*7;gBpbd61ce;!82UNkPTll(l(y>|G-K?z9%5Q|zCJSdbU{YM z#iBiXQY!CcnrS}Mq`)(_vTpM&a(gU9DO#lMmzGcyJ7il%d)6>&d8=La0WQ%Pi?*F? z7QL0megv?~HIfA<8aLDq>8}Z6NEOb!ExLG?+T4H7N|dky6tMjy=b?Cc!gbDTt<@EJ z;;vYq+BUT2WS(UEjI_!drMHLaL+L?F5H=$K$Ps$Sc{vK?x z0`I$P0ojwAzMGq%K??go&KJET#o=x|9Q_QepmQ4pc);`I*6%_JeU7DSROf;sU6(D{ z9S~&QrW-|qA|bsHk*xSKh36Z<1a(43*gPS8prVvgPHW#9^qado0+^TXp#UBh2fdLz z`STvLm!lt8e+;XS1*qBGpSqIw8sHSBbFtg2Z_jFnDw=XSal3B0^V+L_r)w16HK)KT zR~&h`8tQZ--bN-9BW~ZUJh>)7w7S<-C}A_wc)5HF;7^z{9-pKhHa5fA1Qd#)1cI6{`L=^j_fFC8ud zFUX?sQav+8x71n^YtLRM`lRn;C@lnW*)hem7;2sfpSKj}6lvZvVZcA(A{Eo{t|9-5)#q!jC#D8imE;zmi9P``cY_e4SEs4!3n) zQ=R~36NM3&p6}*@)nEP}!3W6)A%L6RSIf0`fc<_WpFBLjWOJU=V~c>ezdq!|*&X&H zG_78PAtRFjzeFGCH;6w2lq9x&#rvFH{qnfS)PuiQJieMN5ZWzZu~=Ae5}^1I zoCa{%Ul&muhT%@0(8{d`I6c@OHST!(`uD45o};sN(=XW|E$Z-fWc;0AGktA~y)vFk=z~ zUdPV|_o2cIed;Yxl@KpU>}uousvFtPDOGOlk3y$0yzuP_)}!&zy>bIOjk2c^x|eU8 zD)=}kR+qms3lKUPQj(+;7$v06k!x_4=DVbehTv>g0igr}5%HueBgeHANkwTD7~p0h z{aNPxO?SNsxnCqJ;yJ$b^b}q6o#Et`y`s-W1x(UVGMT4fhk8S{UCU)a*#!JDlb3D4*}VjtP>7kv8ST#M0ymo`n&w8j&M9FD0!2=+--! z>a_WV8f%2uQ8^5GMo~GsyN5}zj||4y`B8n0Sy6}>w#Zt27ipwiSVYqN@W14DdUlb^ zGdx`-jV(@mlBR?8sr(%z*IxB@hR^#0Dk-!cqfjm|9?gbS2K{b*PrYvxc~KvL_#|X~ z6HM-eG_Lh=XTU~f{n8?ZB?y7^r#w%@wI%h?h5i{4#t2G1$})ahgM6-^?( z@hc9|V^ZG3ycB1%pDTx*n{vnsfEc+FmIXK)8Yb7~DsfN`;#mN%08)f2@14K!6Yf!sng zVl@O?pB7c+h+x^u_IYIH9MAiX&g>7F30M{+cn+WZG_at&dIL!OBpLQ6ptKc0;wY~d z;Jlum>u~A2E_Ik5xh{7i>8o1nQ4Aa}btCmCKhPk0U9rYA-L7PWj0@~kELxM6B4Z z*xEy=rjXxZH5^f;CKE}uzt>bn6a5rb@H_RdThUZmU`lUY#DrzXc8l_4E zbF-ou)%&Ho2F9z`MgzTM+^fUPf*y}jVU%hCD~WxSUl?;rB#_XgrsOwZx^dif9R?f$ z$#$uU2&mR+K5v1CXGRMM$E474b(Kghy}myw8Enlf!%BU3h$Jl=`A7583fLZ-MxDOtm8|5FGxMIM`FG99j)%{H>TSrSW*@0{N ztksF@!Nt>Z?&X@(&gV4y-3#ATUEXJ2lw3{oex*}PH;S28Fi&KcLCGewqilRvqaRAa z;Y1&?zNm&U7mzeJy;A{M)DH)Iv7>%B6W)Q4mxX%Bnhb+wKCBJdT|%NJK3f&wv2MO{ zCNpvX^JWe{!&>-lmLa5NPxFiHuvr^lmbKSQw@sv8mOYalCX}y3H{EVUD~kHpsSd;C z-{$wPLy1>{Jg?^+Ya=4@Z-2^hNohd;;$rj4cXy;k>lY{QL~%vI9y|A+fb4ci8%5!= zQ`ZlUJurcI6U;w3Np?En%?TSZv0S9q6;Y+HmD$13qG&QaBrB~VDIh9d7Fp1>QQ}2l z0Z$29wPxwAsdg)tInZAGi;n0CW^&Y4=)DU0DoP{|=7G|scpVY5ZRWe><`8vKG~gfQ zF!opLY4qNW*SjkfdEW*eNB)=8^MG*5atmod{DI#Epgpm?4Io(}J(dnR{RI38&e|8w zoyD4JO1rw>jnZN$QR;?jTxxMm)q(}%j=mr1^Vc`}qNY*VgJ4kVn7*#8N1*;n?4EC5 zq0(n4DA4={T*N-(JqK|%iOxJ7i9^l4JfZY?c#K$ZWepOTlkW zxlu*iAHqxw7X)_PWz<&J#4}e5gRBiFVBl|3w}zM1haH@8jR&!ISz-x@5kI=UYYMAi_Id?w$m5Umh;ew??w4}0MU?X%Yq4u>5jY}$K`s4#owpgG^ zq(bU#EQ05*dSPsLVJsu+6^A(BOy<>p%l$kvV16p1r9EfuTJF~0?*lw{XZ~1%^4xs9 zOxDY=kCuVpp5S2~&R0{BwYp2gPZKIj0!t4xY0Ei!q>3ZuCM!NTSUsEs|4+=)>4@X6 z@Iq5ci?(`^z<%{Jw zW_dNau$$vY@HOEMXe8k(FOoHmNE@0kZcivfRg0B0=1gZ4wOL;3?8AiO$sv;nN*aif zD~VL~R(t07Nhtr+7@9%;MJ>kZLIdo(2v5!^j~qGzV)L~m6y>l9Ixj`jP@gEAmqjm` zbATReWjH4`qc@)>DL@f$Nay+Ozd_h?0>Jg`^Wpvk{*3WqPK1gf9*)x(83<8IU|!!1 z7wv7*WgGO1G{aknT|{mN*aQ`rxIvO3?VcmuN1Br+kMp8mZ4+IykG+gAsjW+1XJ+Zou{-Cwyo zeiak?_W<+!5tcQ11s|dHwt9s-8e-hnb4z66Na}dLsNF-_Ubw7KMf;bEnr?A2MF!R1 zO`%l8A^P^u16;7rikA#^AN0Yiv+6yJJoABzRsOGyFT{QT`c|WsGY44@0XtB)7A5G# zpgAiR=i&6~22l3--D-Wf&3xLKyimqkG}Q0n)*f_2Y*us2C2d}nA$rI>3X~8}Ynlrv?I7u5K2+}TNMf2H z=2o3#*;eG|yH_m35l13-mi*%>ri(-O_$cpTWW3EYWpm)nbW#xUFn0PSI&7>@O2^1_E=uH@&MzfERuqNmja}@H@rur$py?m-D6j4SK#-IDvTaKa-or z{Mi;1$Oo8MB~i`9*iN^66FY3$@tB7I^r?83iN5px^Air`k=j+=1T^o%;e{@9`F46geW`9aS4rXzP#r}XCnl3PWZc_ZG9R;rq4T!?p<=Y)W zzn-4!3P2gon7}cXmqP+%di9N24Z8{UZZa0w&kG2MY%&i_YRKw|YgMm5_NM$`7F-C% zmj^K<9He=hrqj}ElWmkp-mp;wrB`H^C=7xZ)&#Ev=nY|-iD|61>8v9sHpMZ-+mB_h zB0bCSO`#scxJNK|Wj0LUQEEPxq!2Agnx>=Ja0ge+hjbNPdKtxYJ&_usUuQfBypba& zb{f}_?HR={rhDbCyrrGct{Kv(zk-%0T#nPDAYoDk@3d||4SXHAL5we&EjZJS-luHE zdn1ZVuYhkNqa}7duvR(LLlM~@Tti4RTGH!$1@_(}D|~^5elgR~Ae?IjL!lhneDrWA ztWLIRY!rh-OlG53<@O$L%_J~otw^HSlY&P^Rb=N zpIHFVs$+oAx67LeJx27uyj++tuL__u#>vFUex@%LoI6JUL!`8)F*cjlA?iS~3nlvkV&-49cWz{5i@ z;b*6VF;4-j1Resm8>+14`rYCqOH!7Sj+2tWv*RVkn^-Xrmo4e{YK|DuSJa-2nf18s}e0)4h zczUhdi-XKC&DA=gIhv*EB`P&kTkS3;Uwktl+5lDm=Nwho26(aY@m{ol^vm-pS_3TV zApKhw3!~hH>)Y|=P3~Rl=@EnTaksPT)-wGrIBjmm4h#_)kNbFf)zY5px~2_cO*!Gm=t{`Mjb-@;>CKu#7ba=(72`cH$!@QsL&f1gLzuI&0&`-9#h z!ji~2JZ+w0d#cq*co!Akshn!q-_IY?ndR?0!-uzi6{S0&K2L)Lb#M#KafGq@f8i%G zGw9`c1c@BBO`1iSK}eC5mPQ2*<{uD7cA^bs3ZqrXyBY40B)5bdvc@e%sLrPpQGUsN zg1*V}3=v~i(jw*YgF5)O6+qo$n**^5VfNL*zrqCyPH=6lfetNV69y{0lr)Di3QIhX zcZ>^>lyEmrS27mmpU9b;6uooQ?;nOs)$FX9-VjG(2(-|qwgePal z^5MtaK^5oQ&LJUYsf_HWm#XcrM_sAW@2Jk!vDZ0Dd9H419&CW;^gMMwLPXfBCk?}b z(z#fQ{o*=ok?VZp8Agur+jnbwY(ct16tn=MBno-*CCb!hdfFEk%T)b zAQh(vl@)>J3jOHxOy`Q#U~Mvn)nN9AR%-IxO9Esm9u+R%L6d1e={Da~7{jZ9l$hSwOSPY3D)Z})mrgtglmneIScWX5R&$6CvFxUVXc3+7y1 z(9J>X+3MtaFW0Gaefcx`_^QVF(C|mQNu~io)g~yqJLLTWR`Ye#8a;?1E)CTa02n6oWMQ+%jVWj|(vSF2tTU0`d`HdgmXx5! z3;rUs?uiW)7PfxL$DkQdDdf{nN~}i+%3i9(8v9k-VXCbs!##v z!)YUSa#G5)2rM$LRWTKr39Lw*g_fzv&qw3I1V1}#o%CS$Y?x3xrS3+uXRHr~=W=;X zpMAx|&l|+#I|Iq##l5&`fWy@H;uJvc4N$DjT#IvixCFFcNXv$-iR0F9Z2vNo>e*)f zYw&>5on+nk6VjZTCn&EL%@X#{A*1$4~UL&AKxk~A(>C;a2QdfV6B#5A+h1yGMQl>J@4%*_PeA3$sm`I@8biw`-e^wb>-%IL}`Z`x*D6 zaXnh)zH^BK?t8li&$GLJpGGfD(o%1{C)cLV%>=HTy2Kklv;JS^UH((WiI<{A$M3FL zF-JA+IMU@cH)ZLuGIIi?G5rjK1%HjGi2a>>J9Atkz{CR~@xgciSMEY5M5)SZY&bL& z^)^<_j)pXx1tD4x|EA;}*c_L^lY5ol7+D~yS>-lrLse*Ub?4?LHQ%JHSkWEPO=n7^ zE*(3FcMZPs)xzfZ7nilE0WinpY0eC8vHjs+qbGH*+w6h)XPc1&+8Yuq@cMqn?Mpp1 zN7!n-t9lTDYnf;FV{H9f!2aXF*d9RSkah?wjC3JS?5 zg2Rgk&)++-e6?RE%$I`k$M)GQ0KR+AAFo1FHV3HPAO2_FMvzM2tqNqzWDcUuuMT!# zb{mkWcsmx0lsB>F42mmCUZX?h-$xlPPw3>@>6JdrIdysLh&B0o9b<(k7ywmZ$g_%Yz^~=#Tgks*!NMu45K~+o$`b z8{=0;HyBS=+>=fO%F!djQBv{)w&KBm@9X=S``(v@)J~FyApBu1$&r!-0im1{bcq*7Y zYn;!w;`)9l{gzg5(eXrp8kk|Zl5=5ZviYI=W8rsMIH^1*G#IYs2R#sRl{0GSGP>w3^bJV6xzg-Dx)d^Gm>6`zS6Uzl#t)oLPA*Im0HKxz9dfQyh|hm5zoh*N zxy^3~UrTai<7E9*LzL@lEZ6#COnj9W1b^Sz{TbK*HUN+LbAMdVytcO^>SKS;oDqjj zoxmk*bG5)$OM-R0wYYQGtrOKfP&`4k_qtr$A}~0|a?xw*bB_|K9z#8qIk+-q(QuiVbV((yzsOD<9}C z>uj8LsAfsTa!y#yRNSlNu8fV0DW7$q-Ka9hNx~3lA3h*FfMon{MU1@_U`ANYSbCgi zSXmvD2E_jY@`%2gU`2YtXUQY?%0A@;>N6RoHP6mJnL%J;m;_01SWhGz3(ipDAy0TL zJIy`?8Xwviqg-y=^1dPq_20bYZ~jlcW*KL+8fu~K`oxIBVO}>GbwneWMFJ(nH3+Al zV2!K5lejrm45eIUvs z`6_S*;@P0AF1GhJS`&AmM78PyC5(u-U9s~lFf%+P@qZgc;?OxO)~-_P3OC~CjpJ^6 z-zVrzlYNwB8ApdE-qM!rTI~D6QzDDDo&!`=!ug-^?W~sb)%2PQlBOgZkRV{eyAH?R zY<`w|q_94~pxKN?8X;V`4dQFZ387sPv&j^<%U+`HhblGxe(k0@8t%1a6HxgJ4b6w> zjVMt+=rj20AfR7KcvM))Xx0RTff%ts6lH8qgJOW##ekuL;bnthGKi%i+$)(%f)$7F zp+$l`SM~mbokp=8pyFJyRN2$5WUY3!s56DOO3_VT*I>?O#S~p!5zK+8SpuZr0ED^l z?qA>WE8op&UnlvrXHBg@tr!I>AWPZ-+OSGCz%YfE9pF+n#yrBj!7ma@!IfnrN?6LzVt zeqUDiHQ3}{Dgi+%s3$)}e!X|C-@Ll!2NfhWmRhwy-KCJtl#piF-jIESAvrl=3Xsf=yz-9%vL0m zArufQG*C*HV&UPQYvD?AbWx^;vxFau;l!Qd$raV%Q%HJc)kgdHlApm~q#eTOs+bda z>)-3&b9;utfk0%q93wbEa^o@6b3NQ);P5i_qH-@mm=5?F>L*c|B6vY^^LP9%H6cV? zt=G77NEI4~S(x+}A#1umUoc`wvvSX9Kd2J`l9Iv6C0{qE6|sQr3Dr3OEh#~_7fC9t zxHR`$%L>XxJayWWjDm;7p|{zHE`1k}S0Zq3=?l|E@;nGrut!Rf1`Z@GPc#rUi;l01#UC9waeWwtNR9EBi3y+C>StRKY(#d& zxzX4WAx%g>9+nAm)R&Y(+=2uue3hw%Q4#Ci3CZu-eDxm&TFu$(fyj9{9TYXM&x}+X zlA=qpGA`TY$!JwIL8YC)1lT6DilHtB6{egDv*-KTgQ+`~v6P{~joEpEzi2ulnXr%D z8x=8HZ(+6G>M&`UC2Tn%C45?8b>)G=hxUgYGvHY?cvejTSo}_#05I`!+dTk&T63It z13!V@@@j~cX*3JkUie4mDa`b2`Z_P%GaVn?ru5dD_euWLxBjDNT@Ef=b^u`>!d%$o z7htz#7SNkmZluTyt&nf~ZNiLU4S4D$)s(_>c~S7CvnRoTRy&tD?dfPHGEBVnL`}MT zaVb&a^U(LeQ<}QH#Lo8pb45*#@DmJGF*-36B-86OS-ih{<5ZLzPC~H$=N3G#k@O+3WLJmdZ}d3 z%;xj}V-m*t7oy}iMC7DOMe%47OpUN`r_P>em_(Q7(Z8lCxgpHhPYFDq&^HLi&%p9l zgGkBX91DVkqG9$f^A9Q!!%~mp8^1J(uDTflsk-;j@1b0fdz%W#cO?-dXf0-Yx*`VE zO;%w8T^A!VQac%7M(L@YZ@aFjsBWUPb$X4_nrtBsSV;}yg(QC_uMVFtuMV%X3}h%M zR52p!n5m`R-`4J5|HC)~S3Cw*>jBxbySIleXl%=s3dUUf<>iW<_0%&5kX;y*%)%YH zEbx#`mxp^b@=im$HQPeo=#Ry7Hq7}=5X`X4mE*0Rz(gl zG&dQWi)&?O6#UTA@hK>Hm6h6%#?y_@!D>z_f!ffe%zH3WkSSq)90El`ZI<`6LeCMP zz1T*dPlg^V7ccLijXBpGzlLFL z>a~;|%1_dlbbQ86-`~Z55n}K5HA^}FO8*h*RIGHOCr}@APe1c^9&yIHLBL8X{*|sB zi6HW=6jzJ1$2Ru16Mv~upK}&ii^z`)OHRGYNg{v#~>>|GE!=JGk`XM=w?8BJB1`;1M0%X1&8{^J@vZi(Jsb8Vy>kFg3WQ8KZzvtja6*sStY*| ze^fO{XVji(@_IK*=(zpTUtN*nYmLHa>d;C;8RypMWNkramG)7WynEOTp>I&>j37iF zuf}i}EaXnnY)9a0lLL5H5j!{^)!`WGWEO^t^Pw;QD6L-i2{F%_-KlcR+U!nZ{41vS z_q~`XE8yRVi}s^9_PLToJKmdQSvAtjVvJVx?6P5;!wAt{$}iSbG0lSWF$`$P9t>T- zq|qqOC^laa)6e$tZY1cbl*EQF%!|Thj@D`8SdEC)N=ImaltZH)ygcftQcTLVwvEEW z;zH-}G<1eHX(5{b0^lbdSnN+75tr2Bv(5$1@ovW}*IU(srn7vH881Me3(G5Z(?s{O z+?$8HMkOc%2)lh$&V7NOp|s-WH(rYYi_>;RCs44j&&3k>cCOvx=?uvcmJ^{66=5 z{BZyOapqx5SH-dtDP1Or!5+*MHrz{tDuGJArnpZPs7(;kT!}e>72}T&i}#SJ{-Ru6 zXN`t0#5UR|_pMHfHP^WAt-gkcTwC*LX`(#xV5^j9nhJ>@YUuTJzZ&8V{Q{Gv-@8cY-n7h4#M=Nk@qp zloFbX?Ar@-(TChfBH*ac&10zBggB)?*4Dph74U($M@;}wa`5bZBq4P#^b>R)$rEmV z=bleRep&ZSuOYr%GokEDQBB?^NAIp#u-^nTxW$$RnX_ghCTgbZwwI}KY9&7rPQ+aC ztP~p8hsG1IE(O%9OTGqx`YWTbVav}YU;riP!U2@1jGpHWlpCo&k%aJvk{)M1Jl?iI zZA@Bm?Mtt_ePGs`TESSFo?bAy#4`<+V=sswCzI2?*K(H!dLCyRfjc3p)7nk312)Lg zWfmT+tkZO5--!iN4+DCG?FWfOByYDtob)JYUY}Do(1%jFeE~IU6*NUs3569}TC-Dq zP={~nd#xGWtNi&H3NiGa$dup$;t#ac#zJ$6TuVQMA2(V?LQSOf5bir}ry`FukD>Ox z!ILF4NhDS7 zVG}53UX|HSkd?eaHu>aGSqeqCmj^NQmFVqfTS+oi7{8;DbP4M*d;_w^W;vBw_FS0? zyvRG6$_Ol%d_yL@Bv!>qwoqFS;#f!6!Q+k(fC5Sa#aDoD(ge`Sdt(b&pLtx#aS3y% zX5W(NpoJt5x=hmm-iLS(GYQ$_``h}3qCf%;>mbZwq0s$ho*MNF zvgGRZu(Kj5!*3{Yda?5Ue_%u|S?za&G9|LXXpeMURqes(aSw+gT-V+Frfs}AtzpX@ zj9K2Jv!m=8T1ZCLcBFF9u1NATyM<$Uij zzD3pmxS0Y;ee{r-oFbXK1!CAIJr8QI3m+{O$WVT!M)Ln65)RZ$lH06!>4S^)^20eZ zUB5=aI>c_1ct&fLpmPkZZgWiopeXV~c|Qao%4un(X??&HK!IW3SIcgB+9^VSA%X*G zF~#kfyAEQi6->atN1BqbbP2%W-^B;7+q1W-j6m)?EKjLWQFboGdFatpVYiFM0=k{G(f#@ucd@Nkb~6ldo?I zFA5Vc+HF_x`5QyZqXyb<)eLI-@6I4$nEMaxFP8j{H(`ymWwrIdqWgSfVYjbshfm=5 zdyOU_HYbE#;`As&{svcuc;Mp`N_<+~hm!~TdD{Z5RA*86XVjnUQ{U_sNY7M>?3_KX z5@=Kapg1+x^Xx9bpa&kb8C;W^fjJM#Yxh7&SJs4qj|e7K^y6!yJVv$a6P=9 z{G*Z4#4j|Vn$v0#r$WlUkdB|2TLWwVUR3YHH%uem@~Rbz}Og#yfRmE zFy`NKXYG_0f+KMx-0M?+iRj9iUZg>#{87SnD<&oJSN^QiH&>Tpb)t=p4-&Ny4w(@S zO4Fq7KKqy4qTb(e0*En_N}~6Rs<_py@Ee0!)!}|h4DVwPV)H3tb`*b1ET@9;XXVBk zUqb9l;SQ~(M3hc~33c7iw`XpUN*~+b)Bq&$o;x0kLyFrnk3S=@ z6S(r7Xvn`(c&RO%lCZ<_kg{%BYcT3~D!*z;CLB5VbWe4!cLsul0M!7TXA5x z%hk%adC;i@n*092dt=Op_`H6wO&9~Rkcf8J9Z?88=#|(XjbLBB8P|Arv{js^= z_L{kL`;hzU9%oyo;JS-e@DSY(|5O+bv_lxK8Wz?~zXoJ5Ouw;s?iyQsrEuQ>Dq`Em zgcv!O8yBLf@|>cZuW+6A0sa1C?30X~QcwBSFcO30RKt7WxIUZq&$BW_?HSi(p@_|T z5CLjXJ#K0}!zaeDvN{FbrX^i|Ce`DL+ALktC6mzh2?F;gMf0+lK_oub8lPPgN`kYQ zP9!l^`IJ)IP4^n%hFL*(la~@T4U^xsi@#;3T|8YpBYV^vCd+C`?n1q>c_k5}h|9R5 zJapB@<4N1YcBL$nK8Z+cUxx%p%3RT28M8qnz!*a!4_#odjvLtFaFp2C&T5PgC&}l3 z17EqscAT%teSp3{I#Q{mNp>?$#hG6#${m{Y&Gj!w; z<9Wy-xxqmUNeVKpQoMOTmESNktA8p^Th_dAkdUb>ZtR+Gv1ss;R@se)wecU9(t9p0C>zt!1{=<&7#yY~_=*%Y;`M!n;C z6Guz$tCXfQnGt|}+l!mG#5H;+-0Q_C8OuJ{_$>Z2J zzvrky?PkNFswct=M|fkgQGE{G$E}`;VcLzx-r{NSLLyZ!z=7UXo7Bz-5{vatUEg@< zG+OJn8&W96Rhxm=r0g}S)W?+2$x>j#pKKYjy!fxs$eQv3x+O=*O#9Id+h^wEDMgUT5>f$o2|-O4|H07ZxRPS3x3zDa2EY1A-g zpG(LDStRzLa7Ig7t=ZTvW4Fh|er#52<@)%#Da~tFmx0DeL%!u7oZDOiVNX8;s)2S# zukM3kRO4EXwu>la#|3{O1$N89zevgW$X!wVV9n)&#DIW4ojKtCpF?1_*MZWgSwt#f zSpw*UA<3*vZ>glu@ap~;AezPu+++b#+gE=IUxL;Jik#$17i2R+BD@0GtbzDQ z@w#7t#N+itU>K7gdA$KS+LSmWeBO?FDll7Jq0G;BOds~g2{5r4h&nx`qzeSZ3PffM z`gY2jroVhk!r{X9ts?>}#(ZfDRQ2FH6|)`X$qIkA9$+cw|e`nZE?KE z0Rssm?IorxPYG+)s7ypJ-8)CAsQEjw$}*Az|Jv-n>mvSIl4lJA5jY7>)gFG=%6^}$ zBy0|9eHQeAlJCH#;{B4()_{#G(CupF8SolXH?eDM@F;aX(>a0R&B`6L0-1~0?gN7) z>FC1HbL1M-+mH0h$nME}U49v;0MuU(OUw+WTI$mj9BDSO;x)V;YnKS#U0IX;f$?i& zC=<$ZJoKgYSC*9StPG*82_M(pcIQq`L1=t=~zXpH<>j0fLay%M`oygJ2ZxsD-fiifR0FqO!GcHr# zKh>4v6JkdeeALHsBPj=qPt`0Vtm?uZwE6^?Ug4t-Tp?!u=~9ClBSU5}WRedc+4q<7 zia4dCC+aD=!Am$_?qeYt^ciKGS&|Ym5g&@q%SS?VRP`iwsgLes9`ILv(c|3K4RFAg z5!otz{4c8Zc8V*b z?t3aXO(VrO(XWF%AvW%_aLDsskDt?p`oDXH#HWIb&iRQxFjWO~;sR#?kCv*78}lT{ zGB)o4x_6^m)ssxvc|S{2HUu`8hpLb=05Ku*On>nHO@tW+O;QDY_ZR;8D5y(^|Cc7; zN4zBF3J2590z29D%ng}&4d=({a(RUI85c5qa`L^BfNLnt zA%6?37={jee8XMk>W0Gd%72syF}~5VyDrq4RPt5p`WHz(ZVo!X;@QuIG#`QZB`kNU z48zA&2o8VJZx{J#C_(KoX` zo#Aw$EUuqaX=ql4Tg{l8En1#r)R601n!S=2L+)XbD0FLNmQy4Bz+#bbm#4MX5md2c z&Sp2Hh-z1R`kmBbqE1KSbRvQKx?y!UCZTf*4l3l9A3$vF5-2|K|HV@*%vmsK#TZXzQ1Wi94OcLI=scAPvNDM zs2BYu@al019xT7j>5^|DK=6W9XxtWh(>C{6v)OBJKzyNth5|7{lLj|V+!v)G-il`_ z_=^6a98Lhh3pVXH3V*R^W*59*mZf^(|4K0)4h`1p!lQ`b{MWB%_WU}m2sECDC6(2C z0@2VeIAq78ZMm3^OhG2Uv?qk8jx2m03nd+KnzN~)=ocDJ6FHBWP`F7$z}ZP~F62z& zCQiVB$I4#pRrX60VMGAn!vmIuCo>wtX;GE{;j5Dq;!fn;hIUO*j1QY1glMOuO{|1b zV<<%{UjM#Jm&2aY*f((%jGMiJ^-bbdCJw~KxQO0XZygzQ9|uRYg;ngdzBvuuvu5>= z|HiuAY4JKvfyb^RQ%shp%!-3F_-lx@e2Vb*5RII}7KvNW!}qd{D=Hgh&=hYg>BP=_ zE15VNSOIG(E@XRy5X%?hIx3q^^{6t`Ygp5P!qXjUr@C*VjBzcWLImK%|Ld}AEqr{y zCgAo-(E`XGSibHy4<$ba4?DSMqh&s-=F|Sm!M{WAYo|^4liMi);AZg{X~%Qkr7=)`FyfHb9 zlt5ECz06FY1r6LBdzox>n;5i@P&3pz#9y{N*IhPksV{yNd&GzD+7(hGi9$c+f9Y@l z=Wonrwt$V9w0($=54BDauOQn#Gnw?-1dt z{pr8!U+ozF)mAN+^i6h2PBM7b#dhuuA)_tu#dc;be+}LGcJbcBZE<{--Asgm^KZ}~ z*?2G5Q{R0$=0)cYn$vKE$;{z6SkUNK09bn9k8fZJoRkGFe{cLQl>1zk&L8=JiFErS zD0!SWxOn||N?>gR((cxOs{>e>i;I8Aa;7>laL04;8i@%J4zg`G73zkC+p5H=-d&_E zo0DBG-O38OJqlYBu~dTLJ1;@jzsKY47b6vEz1X(NAb0XrBfBR=Qdx zL^AV^7&y(2hKOf6mdWjM3qqXDc|khOj}ecfK~x6Re_$ASX2>G?fXZdlQ}sBf`X;ct zGz-~*8VY`qA4MS#a*3#hVK_vuD2{d~h{)~Ghel)YJg>~OcMajr^bNUrp@qFAW1sr= z({WgC3%90gTfo?19I-U_d~*7c!GY*B$kAG;%N*sw?05V@9p-)itKp}s80C62({yeM zr2PClF!hO4=(muu0Xy9Tn%@x?!$uWUKuQOY-s_87fUxB?fcuVWr6=o_k$qQ?cX#>M zKj`zgf6Q3eGSYxqn?6}zA@E$_Xh&m|->Kbq#=m;(Om4#1&c>CCM{>kH?iC*H3XV|zEK1ekmU zI14De1t7oMeYH*s;5Ya-xTQrLHv#a2>%YUO@j)d_3$Z|^w7M&H*KZh+<6d`V6EZ8` zw4a0VdIsmR?XJ6A;{TLTM&?Zpq*50=A|sGz$43yB!yz(ICAF8&}JKT|_a5A^+8pMbG!{EdleWWBeeQ86> zDww*8HoI}qzX6Y3rC8V7B(>uwQ7-3WC?2$6EecvuyV@5d@qB8k!XL0Tf4*-+rq&SD z9hJJ{NctTb<$UcB~mUw)-z@!njB{Arxi`opU$M|0n*GP`bW@3lbw{3 z0MZG6+>z)!H5H#HbR*ok;z#U3OI9M?+`x$CSL4iC%fo&5yfRJecyTU7-mst|#m2nP zags^0Pv+`(JE#jlYfYNN1w8+E7jt zyaSkmM|i0imJT62a>}QT%GN7Aa#Cy#*M^+bYKt}#mh{|*j#h*L;VaQlsh>33eax^7 zgKyPjFv7s2eyCp!9*YU%VClKJY^+G%?cg<2a5;mkik_<7S`FfDa+#)qhk$Q?0wg~I zcoa5){6sm;Uw~{ctz6@Uj;kwEC}Y?*S3gJ-VwOzRm|#{Hq}@|NY~L0=xJBkcyhGpCN17BWMG9yW8}#Ri~uA@5g7s> zE_-=ng`bb-|1(C^XlXtcnqK!*Ph6YE^=Z9fhNTL|@?MrqP)zPGRu`Q4D8nyA&7rZO41lniyfPMPl)Iiq92|?R>i(i1N43FGPC>?BRTUjb0O_;xlFC#KJ zRE^o7Slgg`WFK)<@62S}B-V3xKgtZ&02lEP%=>s@Lk%N9yW}U9h`yJ)tSr^#EuE0G zu8vP=fC3mUQhOq6Q3?W0lIV2)HfYu5>$nh@Q|)kOAX;0d@76$jqO?T#98`W~}$)FnX}8Q4iK= z>C~ztz{Z2IliPxifzI@MJ?hGPEcuhi!KCISQ;5q2%b~raJhB@kC6b`TE3ip~tK*#M zNiOn?FM$uHm;kPlpPDXu->kc4Vk}uD`{xmR0fUVy>M7RNm=i0c*Lzhqe{;(Ix`L_T zA@0g*w9;zU?;2t`{?v3C99u1K$BR0HL+RR$`zMi93FN_+VTAdw@`BseBYUdF=$Fo_ zh0MSeC_A|XMyt6u!(5#Epyl#DY&M(uJmh3-cf%e(1_>OgUj)Q-NY-^yy68DH6%4yO zuz`<7T_qzMXBtD)iI*d`L74OjRk2za7MP|(JcWgks=$hv^sredAqg05L&gY39to-T zvz(%~hBu;OsEf56O2qGsf;?w7+S6<5#`KBLCax+fu}IOM`0|vPXwTh3e6v1FCz6`Y zwm&4>&~x`Y%N+{ah{bWX&UtP=NnEgeu@2Xn=EyY z6n~a5ZU)8kWN_5s$wWfuO}(%m_oM8~+ok0J)4`Nb4F3TY8A&D!FevTt64wb!NQ%tX zh()&RHaElj!2ja%IFMq9BHaAmwGX#6(t;$G6byq5;gIAaHvsI4x`pyab{916DFB!S zKg*JYSpGfvMX)EkhJQIm!E;t07E2MWhJBUcm-ZGbrk64XcaDt2@spVViQS)qIEpt< zB!pG2SiH_%T`&CVH5+D;MXwx#c6cWBAESn)CU?bEAV{P$SKg3@B;&T02wme0B!6(r~xTgim1p9zX5T2 zwYtHzKge|?zxYaw$gzf=wF%iFiz5pglf%|zGP#P}*yd-na;wqCw1);tO5>~Q@ugdz zMt#TLfs1mVWcd`p>n4q_@IH`v(2rBu4G2)P#_xfr1w78-(|KgTPFT9dLx_yQE9vPh zlvX;qO!>bJkXh)d?pLn%ge9jE8f&_ml%kmb2s(4M3T?OAH)vGYck4)MDXiQ6?FMa$ z{Z*&0Lof|=+z?he3ILg`Etx@P@FpB!I#dHB#Lw6VZX_?vCLZT+dfi-G)h2?f(!ksa zzI!;seG7xMSb%a$UDGZb*^<+b>0s4ur%e$TjoL{NFA{C%kE3YNW%5^*?JE451yd;L znngZ77}+!z{}#?QAcW*0*9vX#8bO)@xkw0M`OfoEu54PK(lkIZB!qvZRMJ{Ui@PNG zhCzn0r0gm&Kll4YGd1hSi9R01=>Y8;WoR#~4rNS5@NB)Wf~R z*PF0=V-=S(MVIHep@JOHoEq4~m4;B{^(G6vCF%q6--xi0^@>9f=MVcaoCDwTo*32xDc+}L zh24PLPQa2y5LNdF9Y~5AemZ(FP+`?tuXT0(u_P?8%Pc_x*w#0k8s6~8ICc2RngCrD zlnjuX`b-l0)I(1M32<0e$UqWQeh@K|JpUiOPA4B|sUuibiqsYd{^ouu`>2eirCW)4 zk9H~V-@qicnXq3YgO4vyZpxw7J`JZ(KBt*|)SA+X^+AXu-YQvmcM zpyVDYO`NLWzy+(Ax&|y?Gf<0HLgCfz_U9CmBxI$3Dow?Y%{96h^pXgFDvuE$ymUJ( zX5BbNDrC)f_l@XWyDQ?9(R_xTyQ%r`cMRFpQfu5S$Q=tdt&Q|9y~YFpjHLk#wsClJ`2 z#X8?#TwNdy(kX`Q?fp@M82WRVI61Ul2rt66=@A4@MijQ$)tIb=Bt1ea5Qmnau7aPJ z`h`YyZE6U!Xo#{XGUpb=nog-O$4BUlWS?b1z}6_}Wz5nvgJ7qtN1_*#Xcba=a7HfL z86vIe611Yr7&4B2>m#Z&yWdG_V-qz6)r1~oYO1iVPwD1HhG-R1{R#G=Spgu4 z(=n1Bu7&t&feZog8p=nSryiBxn}*1ZD74O-CuQVsPZ+w#p$)w)$teGV?T9OcC6^ZmlYKgc5WHGjQbnPX*b6zw)Ve_%$WHgh41 z+5m9b0!=TBS1{XDwAO|{yY6=ziL)I z{Z>kBZjaLIa%#&^hd#$5GHB1Y?6_oqA}CX5n!O-Z49Ocb(F!*uC=2^Fddi% z5jc(J2v-^bEn&#cTm@zcL%;l-npLz%m~UJr^h8xocaisKy1sF{9xyfv41|_~wKDvA zR(_WSZuhO~oP)seWA=l9r&-%4&AGCAWggGWaZ{gwa3t3~ZAH>DmdvY*a5Ru>WpsQ-HN#Kvd?Y-;j1{ zRO_Sr@^$(LQ1Bp1STjcq(+;Z!D9Z6=CLgRNO&yIyJ zVd4KMe1@g`y7u{%?bUw+K=(ur+jZ!97(n!5_g^EsWsJ+32G;>F&1NDNgV{`mWq-F>)W6 zaDC=U_&yJ5eZTiYB3oVbNb+{I#ko^lsX@~_E<#dCuFd=XBmR@*1tiN4d}%BbfrpBK zKz*-hUG{PDbD4yf^qKC325Dqbzr37~^%YhP9imMJ5Zql!9&|~E%m^A*}+01UWt;1EhgLQr} zMD2tlsd7&FXn017D0QLx3(@j?4iM_IobVi(T?Jbl0ODE$HMgD&zOoYS0+?7zRTyv#a0 zuDry)rTwbqRBzw{>G9PR4WVMxp^nKofPqGCW5hME!at`WO`!_TDYWY`D=87Pt}ZVH zePDzP7?dEIby43ww?%Q0-uwMq1D1$C?5}AHOXpzHnsPbDxNBnQulg+JAO6f5-MQa11>NSKss$J7-%4#K!|7IbYy2}WnDNDeXNgZiqlc>rBbIf*T?}EmR z5m3+f6j?=|{s)B03|Ww2ELx(@Eu{ z`c-5MU;5!>V!{u6Q$ z_d6xxbZQu#Ig)yl;z4X^pfNcf8!T#LO3Q3?mdk8fTr>UI(O{}ZK~tDo-{TE&qvLfD z<3XX%iHoEnk^os0*aIGTbZ_bc6|xaJyivDc!2Oqn6c4 zvPGqOh(f(~*FTzW)o{WDe1CmMpf(uJjkzJ*8Q~UbQUa5FnD7$_^m3^bWquVXWo-2a zW|oGgn0iE9DZ=TfrCK^`Wzcjke_CqdPu z)2vVvl8#_$X5dF4aB)YJ2_v*wh4|0ZSHxjt#>E}pdo7-QSdqP$V|N0+T@gF`xNbzU zBpFXLp1foBPFL#=+J0Srw<8Tye1*h-K)IK|#B~}L@ ziNPQZ$!NYXIz!lr7FtVQDCDMPHq?SQLUoiZO;{<+Na7QOIDdopZWRe@pYT7&Rs|DH z3Yuh#M_Snq`mse)Oep=|&Vx(Rjm{7f@;hufGWZH*dT~qFHHrLc6*efAuhXp=yn@+Q;UpaIcS5D?Yhp0Uu5>fepl&c%bL1LW4tR!x_Dn8p zze$?vllOg9EpTR?IcH1>s&Elx0GO!+wHX57;_c+r+Jmde%}M3i9Y`2#p3^ zu@2k}(Cza)Tm>Ep%{=3li8|;_Z9%IOJma=9iq)qcbF zs{;f(cC1m6Z;dGoS}?j1nqXKP7;h}rw4iLAu>Ki74x%02ve%3qOJ%6$5zJH=@DcyF z0vpnCHi_gV+D5Fh_64b5HQ+XO9YV1!iPdg0Q!tYl8Y~tnWM*B|LO?*#^$U{v-jPyx z;kP2OX!Do}v|pf5zt@gzDrT?^DxU|z;>=r6re-QQ-BYgGC`Yr!dDE(ULg1c{s)5mG zBs%0X9zv!}WT&G9KibQQsKXf*{Q&ALcBgM~M?k3qT6ZcjKiWFQY*u(EotCpl-zbrD z!(3PCj|Xr~3s&}l;PMU14vp7DY-SsC?V1H7g!aes6`*f*&T6s5P~M7+o#`(k zcJ+4)LEYW84rr<0gF|b6c=87PKUA!P5Hl4bFw=aEQLYc-7|3Fb zC*`P77iD^W?zDOYPSfcq45COpDyWSqC~@k=90m=^FJwO++)p*y>NR9WqCX zuHV%OvyAeP+1NBe=9U=0F*9u0rEoI`q%i$P4t_9&G&t#4mdw|nC@ZP?%cpt)f&MPm zKwlT846CECO*I0asF~u#Yv&1a50olMuU(?YnPg);mwe1{9m-Ms{SBbQp)o4$g`2=d zY(`8@C$9KbakGSxhas=v-JvlpsxfBg0rgO=$4DIkNi3Z?l7)r&Kq(qno zfD#TuSGK3V+&ds3a*B+lgnZRdT%}C3h)d*p_NzD1Ki34R zkz2lfZMtWsSU5(!R_rMYBZq3c&;0(lQps-MlrC7@2EPP_xFUaL+6hH1{(dM7+3-Ec z)PC+z)6IOwNk$)}m8eJUQ%xH3w(iwTR{O`1u>y^rZyI2%7t#M)@sv#M9FLj0e!A7$yn3TWlyJY=-ACIg5}UR>`(h3RI6j zHql|M^P4MXKM^unQ=D(__&^$gve@(6XMJnKrLHRNKX7g$HZAbc`G!=AN8CW-(a1`Gm#ir*PUWJdNb!wP=8gx* z{S9%UW4_MGkaAgF1QMg>zB9s2mITa(hJSHB6PQGrwZ$T#-g)onTiIhZgbt(jLPC&` zdz_W5+?U*HoEm+p(MEv3mGtRm;h2Mjpuh4js+(Mc1aAl5Lg%fUjgXb8|6Ne)G%`5g z9l*p5x;tiy@o!)dy0L6@PP3N=J6I{n(&d?udc!p4@b1$jh?&M%+>kJa2gx)=T+r_s z^dKwQ2ZcTnNf8ET;Ye zcMg2nQqwTo^-9oGMD4HP8e<)$aQ}tdr*b5EiWaQz6}SU=l3nHpdDu(DwY8=V_0UW? zTKDB(JPHKkyd&#yEr3Df-2b>*0ngsF#|>sMQhA zkbiHDT$gqD3mx0mTG&nFzh_)=|0EyI^5OCn^aAqWPW@NwHU+M79bg&}mHyiwu>D%4 zVVDc{kMA<&`&=M2fZde@>e!W~*EAlU zeWD4NGXh?e%<+Bd-vzlv%3J|nexu$y+|~`4MM&PLj`ReSCeag-LQEPtgSFLDDa@ud zcs`X621dW6$sO=pAt{R?Ga>UUz6c**2f+Je$`_=eQ(wV><0c*& z>By5>NL)yvn;J7uCVxfFLKe-f)uc%OEh$v~ILu+nsKkk!IN8Z4?N&#!H|eKOk^Gwr z%a#J$SPemJE3zKu$F8O}0*C!?mt;maWNVp_tPq>%<4LKh7jCy=cm#LB<|l&>T!~D~ zw1Pr$n6RdII4=C z<3DpVUp6M&65OWdQyI?bn$dhe8wG^-ZxMZeC(UOI0U0=)dETDLF_|lFP>xaFZXd!; z9{~ZMH@9b-@c=*H=R$nk9S9(>&oSc;C`{1&ENpk+1ykZfLc&T_{NWX3a9~Wze%kQv z(Lk!9C#nM#YS=v{>*g>6DU9x2zNJdFghA`-1*Y#BFMuJ#GKju^2B)RtD^Wv=d*7SX z|GY-|W2T)iR%I=N>3zu(F{<|vdndz3CQB1LK0&tKUf8^B{QXmF;WK@ua%wksL52ca z`2ry$Ar#$501SzKkl{bG8wRXcIlp1qp7M(rmJz9k75A79NOa7a7yt9hQ&1G0BQF0gSFVy^D9Wr)1br}RKel0?q7#gv2 zB5qG#PV6yBIgmIlJd-rYOt`7(l_FUgQ=Opy?RQGUqyeLOa3{B_0#WgeP1*Y`FkP-6 z6JY<`N`m^@Er|aiOa&}k`!Na(2YCtSH9m$_v;W&ICX5O^KW$aUX6aVRq99_0Pa+JA zM2W~OYzZd}ALUuLoewx)V(57ah{iNQ-pHQwRFTidZHY&QJ(+gt-N^H6Ni>$hiQ13` zjTXx1teh5X5Z(3T83~k(kS`2=JxKlI+rVXVkMT3Daf)oppft+2lnJ+LHGe83AsDQJP^b@wSQge&EySqCZHKbV8b4`TTGjMK9NP0m}?q1IwFXp=s@ra8; zl!mDFTG^O4MU^_&ZHYtKQ{?ayMVq3lB#EiKHf1{j%EV~lTA(Fwtw|Y^(15h6gAcdb zA}CAoiXHDr?DcHzf&B#e>k07G@$0wu1HkMxA6*+zCsu>lbx&*NaQ*bOL@q>0vX-2r zSwZ7$Y8)FSesmw&H6Wy@H9MS_r>9`%tEmn9T}maDGtSD%;aVzTtbF}u-%f&J&mews zp(8m_cQ2n%^~}-88re)JearNzXpXX&H}UK;bdEpSkBg>XS=%AN%g6uo>8Stpy%k{j z#3P%SwX5S7aT~O4BozI*1#5bc2pg2mVApE2Vs20?QMaAZ%T=gy^=Y-bNil~y$~o5Q zg(lVPq!|$l#qKvo8krIBW41BwOc%-+;;P<(;$|-J*(I^wa0tbRP>dS2OJd{&SJ93) z*=jCDXElnY8PNKi7*Z*wZ-Wq2U1WOk6YcN*3zv~65iAJ@{rKuNYksDO!TnT%LCjgv z&Y;-8=%}=U?irje=gH8aje#n4`P1Sx^r=xnlimTCH4fIbI+3A>-=$u(0*J zCV79MhR0^8`pCZvOYPEi=8e>(pB5>B)R+oh2(Pd~=nIJaMu+?LISWqLfE6Qef2%e_ z?c)PtImT`y>BpaFS9h{AloU(ggASQ&BKs%t*!?k8P8=`D+$$jJ9b&j2W20$S(u)!= zY1_gG{+Q$yj$ykf`UIEIHgtd!oK(}MxOBvm^b&oERy25CUgnsGeLf8XNPljp!Z_i;JgT0$L;3~!_34tQA4{1=*{-J}rCdr|NK*a z1B^`K<>9^b!)eRQc@(l?v3Hyk?_=6*Av|lz34Hgzwp}^DzBFL=y-cqgmY3ul8AUZ6 zP+ViegW8(2HFm}9R*<5X53Y5GIUV~sdquVIliqg*;XGLOX*AT`z!>0C!S7hn-P<(g zN_<9QQ9GZ_bkrT<;(&X*%{*a;-lFcwNZ(;fHw^u1PXTBuu3gA9_$OGadI^dsFSSp1 zj6Y4qaH^3aU=Ju7{IF(b7e0AJwZ+7x{kg*brRPw;{x6_@{~rMV!vpdD5&`(_W-IN+ z2=UMk5lEA;y{fvMc2gJs-VzEV$Z#-tSsw0Ic_Fpg4N#MRdG_#&-`;j&-$wfC3#dT*?E623$X_b$v*$t-i{|0Mf$cS?Y59S4|s&ZZ;5Q$ zax729nXNI<(k1v8(8{)5HP?u;rKhI5Bt=4tGt+Q-{RD|gR4LPX4agDaa|&&{0dB{BQ6vNAeLcDV*G-b@q0jnF&rlfVh|ke z?pYT%E-Npistc=!hvtmu!2o5jnSdx74$+{H8ZlGZc~;#SVZD!O>KQ z`){0fYTeT@#H@LjCzru!g03cSQ)Rc{s``c3+a4E%fvm#8Tn)Y(Hgg@u6vt<)Qtw{% z2nDeWue8wi`g`Qe4Fy!Hcj?|@^qH;ucS-)(2_hWF$xQ6e;HkZ67-^!vV;_sIJJ zIGwbNQWoJaNj~&DUU7i;jW!an3Z9M=S~9ubn?O!c;!49zF$|@(Zb~%EKmcDcBd_|F z%eJVRrtKVaYq2ME^R{u^GNTf?xYjK^jS~6qIM#~2!cT6AC`Pn+mli9EbGHiHbPA`H zy+slJ~I6 z@m?~=#OmaahQYRK2lzLUl2yaB-&moMpOXe5Emai1`SXc$i2zmU{9v3dA{Q!PE6dz5 zYSy^>C!7uvzvmiKhyR)|Du)VDK_2v@d*P=F;ZCSk$uB$e(lIXiJG8o$&J%N=a{UI{ z8}}F(SR;3}mNM3WHlSPS|F?H*HAtWB&gE==_P7~T3SEVEB*hI0)MCeadCw7QXdxv% zrM(+pzwpfOaGX4P=tuLcIz`KKzFgxRtuIn6_4?@P4(QT3=tW!o??qdqKRRu<;55J- z=`B;7?Hg7|p43n6>l){lOP`A46=ETArUv^vwC&6F&BkTpWcB@Lch;)a!)&dqAuNe` zWjp_cA84i42K`%pr=SNDJ1r?jkB_&JEjMm)B{+~N?~9W!Csr0;LHqB4QaXQCO0JK3 zXeU=rJay0tbG4qX6nor1&uuf7LG!?SQ~$fYRfAno6;cZ4v4DSAA>vO!;cZL;|5CQd z1vA~;^51hp&PC+;jL`%rzv!ZvW|3zZqc+!dMrcFBAKUY|2;=L?4!i~hdN*?&O7;NS z1NtZf-du14!@aeAM{mThx(SF`Msvy@F_Ivbs6`Fld{VG1Sl`Tv7bf#xo|XJL#T!Y~yy7hun*yH5x}FpU6URZHUXDquU}B)L9P{^gdXLNtat) zLXWSc$8QC&-z(ozF)Q$bK&p3V4&qb0$#@KLgQ~|^1l39~u*E9)iI^)g;vivM$JPFl zh+vEw8xbNUX|;xvXBZj=FBs{9d3sFOte|v)F0^VSUsCAlh+sFWQ>Sj3IsY<$w};j( zGSIQGNcga)I1(-N&wb!y+*fVvUQI>gZHz9`)?wuu8TFzta&xa_>?7c!`_-9{dyShf z;GBVN!qHjBYAIM!rG4Z*PTh~ zD;=|h9#JJwn$&6DqL8RJJFmMs>vHgB8skC-GuJ=IMGZ~IMYo*a00x3atRs2_+vgBo zh;ClFWDfYmMz|x{fa`Nuulwh2sDn}{Y-E9n3N-2TDYW(vW3=U9%|RJ_jKL2xY^Mvz zC902$o=$bO&vO9D@k_^!vQGkyJE3=G}e|NO2^euG`4ENJqH~)LMyU z8X^BeX$nnVBI``KjFu>7!I9xKC2$Nr=t^8(88jWbDCiHxYD2uERlZv8G-&dy;WNa2 zRSBE*RIyxUW%TxvF9+@E*O}WH+qV;$BNbX{EGnQ{nog{t2 zUr-S6HfgzYPEOo`X(7uiqzm@!gi9Jz{eD|Cw>FJ^pMr&a2$;FCOP>`KrT?a&UO!X^ zqTKiozu6_@+L$yNrJ*w#^frf;GI8hOJjz z4FDV`<|)|cGW_g7h14f)-!9epiJE}s%SU@f>C21G;-iT+QnEEtt{OpAk$7Ms9wONz zh00r`)>FTz3Pv1?EtqJZTnB}sbofuUP%0I_B-ac^)vi-gjyV?RzFqpqZlqJO&O872-0hUVw7%G(zxsii_135Z?9%lC?L*&9+8zrY2WGx4ghe0l z$uc^=E^sn`-`5XL=I{GNaH8Te+nt#t&|#?r>TLV*c+1l|Evf(8)${dApzTZRv{;wd zDS@^)GtvTmU%v!eU4EgqioI+fWF?Rg8e@b|r&I{F>zyq(mv^>Y3N5Gq+tu^+N}&b% zy)G%VP?y&!g_hCrbxEQ5`@ViDG=JYGN+A{D*KQX`p>#w*+LA7-4s*HFt~ct9I#BhA zLv(}Dt$0^w-xYRBOaYC0%D67|k|TJHl<61$DQu+T*&(6qJ&AmF9+ChK#Eq2lP=$T8 zTeAwzgE4}$SqE>K^k<9vZK$Y40H>;tT&F2BbrK57hzteYav6NEB0;Dx%pP8qz6j;~7w&)@g;%i#I@K2Zj%D6n=u3M@n;a-&31FTx(HVIL?l1k&D8 zLb9nCeh?xLBMOKRLfHqE^)<=pzcW#?jUviGPn6V+zd`z?5|JXWuYSLZvk(x^&Guvj z?{~D}v4_A2`VsWJd*FBo`p5-Q0AQb^4HpF-iAAK3Zutl~-Y|hdOF7E7iyY;-;H9PR z=sH@B^K|?P5^(4tZJX(r>gwz7U=MjOW%h^LA$EoUx~^=Xq8_P8-z@%_f{#nTtty8? zqCT<{!hrvxrbs{5yDxFRyCvZ@AQ96q^vE561Bm@9z`lzDqlZhK1CTm`)IfUP-8j2SXOSG0Ig9*bp*Jb%2>@E=~K?>?()0R=(5~ z2a+@)9!IDGu!*j5eh-P~p|GYaT=8$pZpYCi?!6C3*pa>KiyGDjeAOsWEOYe*y3-dW zUWvqb0r8Q~MC6I#EvkA*!A-N?YDtfTX1&!0A)*0t7`{Q?eLZ%Hl*^p6t>ZXC|O{$4E|nK0#ZJ{8G8U17}!KY#pqe#sjZglRTfoujdz3?tvqezdJWnviVDZ=Iwj zQTc?l@&!m-Ol5e@IdLzM6NQ-Fixe(=BQuhk^pFGjf?-2+FXClf6og0?DM=H*S#QZe zSlv^(?g}33B=mrBBz93h_!AEFf!r9?2?^Z@7n2?uLjGIGd#h@X7qUl?Y4|?gXRBro znDl|G9`>M94`}cvc~Zt7D?$|Iwt98_BQGAJtgs`Ml%&vYva$Ks!w zJ_?IJ{YLd`auAmJba4#U89xs4kINu;KA?O%Ua>)*lFxd zBVo#$FeD+LMSeAj&-y@I6n98P)<6nd;)yAUMFY>;^TfAyQLBoNl|6;OX>PgZq!F*2k-QfRwmw7 zY3kiFbpeNaS|gwl7>$BhuISEkV*84gRY0-tAa#VNEJSd`A09#Zj(Sr2m$_>^-?z6s zByBnU->#mo_k6!Vzt?rXU#QFLJl`*)I z9KHhbM1epzj~ct+zd*N4ICgc!wK!p=un6`+AKr*StA1LI5TE~Z{rJB`M|y+}lZU9( z;g1mXsXV#mdaqRZRH^cZFbI$zpXyX5!~jy2Nl2LpLKdGldGG!28E|pP=lwozk-xyd zoH%dQbkRW)-h2@(a`SOMKMzEf$D3MW8#M2{V`&M(Q4SlQt? zk7E)QiFp2!KmJ}lU9MP3(2q=Q$x*1zdQ|w~qIdOh%NN!hgPsAx8ACVGc>NF36Y7I2 zy=>B>n*}7sO{K7*GJCaz+3TDGJ0Ffvnt}KgSFplc$}st!?tIeuNI7g|#33G#5qg;T zNz5STPuDTo_aH-*$v{vNh+t5WI7`L@vJm$B*imQq5%Sr-ZnYM-y8oU2WG#PlufDVM z6?_$%HMt1BszZ|aedlZKYc2Un{l4=RT$yO7ZvJr@Ag9hL`T>s675ECSZc$JC%ZFH# zfAAJE>c8|m>SED*rSn(VkXpc3V0Q>Z*C8%)_u@w5o0*1o5qYit4SWW07^5x0eNyt(t$Ful$z<>dIQpfz*L6dP%}y2z@E&NT~PLh#~+ZM(9w4 zp^WdjCIm(6>v}ThN(bL+#xh#!DdGfp@kW$v5Pj${5+b(_&OFZ)a>+ySp2hE8qkD6S zrZf5`TcL+Xe9iDbxiCYNQK59=o2=5oYi){99>#-cDg}5;D8N;kU(jkK_VJ?Au85d# z)H{j-je4h5zpKkcE-pb-1P}399YKa2u|{J?PzeM@{E#YhM|=R4PAC??l%&Gg^*ga0 zJYM|DATHGPrRzt*!DR!$)u;F0%pPxgUnd=XH>juOhuw|pg40fw`Fwo@#Ne5yac z<(o2iH;+=XaHn=DyUk2k%qk1R_T>FJ}%lEpgtsIp`!U41lJ zvOsUwAWIhL?L=9kJY|}PDOqxu$db}T%OZ3poexOY2T_3|H4pm;z;GZ^32L>V)GMmC zs(>OSEo7g19vOguP*#?fl0UHDYP6dBjf4GWdq3Z`+D)xhYsf5=*<8=s9Y&XRa3LM` z2n=0i6jW|9$W43LH4nJ_LooC(3R6cq<)qW>q)6A9P7EK$V)*D;is98~4cg2eZ+c%J zTMVxx7tls^@^QuRYIEjnPH!g)Hs!g}JV@=%2h;7&op^UXe3rZO>NDzWW{)?$uaCVu zuOv^-Ms@OWcjwjSli8f!PTZZ96H~LD+MV0e?al}B?%a9Y-FY)M=hY{g*_<9fI)^Sb z3r=RyE9b0cE4Aob(=GaTyyy>}+oD&on>9Z6rL7}Xt4lfZ3X|^$4?+=)$4DGBLwxTZ zh;kA)ZK1&MAj{|hl=o4ynVQz-bko|3r?t(eH7*8ni`h_5JM;l`UCc1?p$D$cjD;p0 zvFMeUD4~jp`T)7Tz8uhj;G9Rxx zzeG-m*p*VCN`)@)5Tk&Ti~<@U2lww~8V2#Ka6cDYqx$p5<_ANh`qA?YDLDEl& zxQy$V0v|buQW)Oz#!2YP5`@MEGpd6haY$L!Y+Ly23OE0_L!{3KD7>Z;Zc5cs`gHN; z-J7$^H{B0s@85ixp@I^q)$~`6>YaKcmcOa-9{YscR;^D$NBFfRt1Cn`$wFETs?UjtD5m(A%L5T2 z0DXYMkf_WH#Bm}n4_xes-CJ^Gx*gy^pTFKiU_@Np$H+}@^4PzD9(GT~j#SsxrcS|I zd0qOmpMslaUBu8_ss|q4L*@)+f{)V?3_elDH|Q_&pNC4ZTYlxVk|$o}TNQ}b-rNNt zq9R+y>8CH*FJ4Hh@RC5gz%-cSz(j z6e70@pqiy2ri_I5SK8<%{Ox-vu6pkxhQbl{kxKKT7cX!9mW0>5EfJC?n+u8O^`LWo zrMC$6eCnax%Fuc8ME(;W@pTGbnS1EY&Lu+N(<_xSPp89?srY`7!u+ii_Ix&C10erg zoepInk^mk1BE?0Q-`M}oN!qKiA(HWaH3<%CIBsvR4n7iLY2;uc-rRd0QrTLlgg!1} z5PQmiiF`&+^Lr5;lG`{UaOw?6&hI}(>cD$tkBl@qjV7;T;v>D)d{4nAoeRgoh<@4q z&ir#v>`aFEQln8D#Rc@8Aqv6!7>8shcxmOposusq37|^Of7!n6k-dUL>YD~A_ zU=X4KWauMNQ!B3XexB)DW4elcBTv!wF-_l1cfjB)=)!;>#p7Voui*C~62+2?lf$R{ z8TCalc4Q^gSPvk)O!8&O@zH@qiEtI(dxNi@0t2JnqSPY@0Kdhref1p87tO3pH7 z%ve_Hi=rN-`lKkq852}SRAMTEuX>P*I#v?)QW}jUPYNdD2d}_`zLXjwx00Dh)Wn=B zuuUE)4Q{y=Va-?d-(-rM9D}?pTlXr}jrn5yPM-hb2uN|J%ic9M-+I~i>h#8$YDzeS z=AYvvRSKaovmxkc+&YsD3Yjc&#>P=`jPNm>G)@{5<3I7hCn5TvX{m9A;IFsh9?0F) ziC(ClA_S5ep^Cv%`RugDL%_E%LRI5=TJVC5>l4~B7gI-M&^jk>>QQR)L$d#w>96FL zD|Udd;HnQjDmW!0n3OQ$?kuZtj}!yqJggFB9sBQRrrlB>;}Z`qkkouF;OP9%xmrvJ8}@q*tC#5_&^&A0x5pBp|5uzxrI zr?G6M=JRG|`1%{>xbuG%F~^o!$o0Oz0IHmzkU7w$qw9LMOmgpJwM zgViL1V`{bR@j+a-Y55qU9y)VJkIS#>qmw$dQofzC4=dr{DZjm3P8$1Bd9ZsQc?<_$ zd^o34U{Zd8@J^n5A`(>N5fo z5FaHVF{Ru!w6(qqAP689`e@Vo0s;;s#L?CVKcA(wu~l$8#GX9K%hISu3R5E^uS&`( zd#?Tf5X8sld*DWDPf6ycDo}DhWYoxVnM(G8(rGJIPLWh{KRI>cL=Ju@;dSjSB$2Pi z_Ya@k`rs|45uZbGW>*{lPaKk$1%{WZcin=1 zyQ;8P%0!bi^4s|<5FnZTvWM6$Lh4{)I7F+a{?GsZ|2VZpy;?PsahgokIXRi4WWgWE zVRi5}r9R|jt)d?%BW^0mg-RnhTA>MZ*>M7QDuFSP0ga*CT)O)!!h1Ycju z%3x%LGS!2+RchGgax-(tTnZ2EzGfx<(WJ2L;oTpZ^tgwBF^pnN8Hnm-( z4&4zJO4#2ep<=FmX|iCl`H8%u;7q>-^lLjizh@LAvFSx?>KpZT8Ix_FZwHmrG5k!LiSfa5#`_O627cx@(~nSJR!^`>xDI+((cJ)+Zv7FJGG@=K)e4l#Sgj#cwy~cjNzOphGr{kph{-<^4lxaSu_-~= zDC4mcrG%D>WpNaistI|MbKS>xmm+&9=4?tSR{H_pm!m+ZpL0;ia34F;ghzl&lu#1c z(tI!?oC&#Ptcsn<>(?kA_{cTifz%mdYH)l!7E$h6@EX7Ap*uiA zXUMl3IcT9=lo^J*t6Ubk+1|Z@$D}=*C07_8p(@}^5BPEz3{%2t2&L)m9x-V+>PKAA zD;BD8rs7~^Qy#$`bEKFR8U!V#!c3K`IkBQ3FSmo zvl)oQ=Riy!CDS$~w^~Ei(Vd;Qrs2Ze?D%s2^aoan%fma zfD$j#i4QfjSDPk_myK953{i$mprP`{$JjBot8&gW`8%~a++exk#3xKckPD$Vl`!Wm zVY~)y1l8{`Yo7d2(2n0CU(`tescv#n9qbOJUM-l<1_F$;M27q0%c(D!%&Nl zf*G!hwNZTNQmEA^gCQ55BlkNM8ehwV@5oh~P-vZ9Gh)g_x3mUIE4fzRIdztabLomb zYiH+&Sh!VF=2o@x=|l^qQXcUY9v^F}s&-R|q+^_Rv5uj&_K#w4pg5vXdl?t;y5#rp zh%?{4a_`a7A~uXFlsc9rIL1&8#j)B}d84|J!PwLuy0rj?>|S^`3kN#EjAuSTkluq^ zD0T#w$bEpHiS?0@gO`wuaJ)stmi_X6CHqVj=mSdzU6=#q2ZV9070-z7U|dX8*68YE zRf?y+^HCh}DM{2rLQh+6GY{fUXp8sTN4wSavt?&FyZqbcayb{lqu2eyY`uHx%1>Hm78U;y8{9EX=xG0)@Lvoh1 z>r0N~=RVeJ`3^!vqY+X%cvWSYn!X|6%Jlq66jVCtw2?<#3_`2M1k4r75&P=QU~D>Z z9R&$~1m~FFQP4i_NBpYPu$}W`zx2>RA;pv--$B0-FB-)V0V)slkX1eewT4_ro4^_V;#nT6OTN zZ-x$)t$Z!(`ASl$-BU3&S<=4ZG?OO$8x-RHJvjdj^f>b(a3R}r>xI)Mx1qRGMXMpT zyQJn?buGAYjM>(!P4r36d9I0gBF|zYls2;H7^^s@Dz0rQ$_+y7XRDgU!A-nl9%5Je z;88s0Y|?!BY+pVfaRe+dN z-_zOurK}?Lot<_aT*zjfC=|YBS0cbbu*(JN$7Ma3uG=Cvf#1HTqAWn9tS)*v@aN8M zAojRS9#5yqEL~&w0Rn$e;)7c7^7~H@6%tfVK}8axQU#R&vLU~+&pW-ZJ6-wq<@>V5 z@dzBw$O@SJOjOsEh3bWrsY*q!Bss^+hz`dHu9QoLuxP<+1ZZqbSF^9T2+|+Z)`-ts zeDJSj9z@?4J^8i9p7JF`w=L z(cY<8Yw8QHJV@yWI6|jj5VXV{*~Ry~ho|6ET=Eorw7xsc;yY2uq8j;m+?(x+4o%!!X;dz)~gfkJf$py4Zf z_Ad^#lo#`FRWry(+E)4yi=xkbL8j#)@lh2FIcG|YBUdH|i08l6wI(@zdH@IuMSBlce*IdGdTfK)E%!GRV zNqlt5&(W-Qizur#VO%CAfK;EGXK{ZHDfOjlophTp6M5eL;n87dPkXoE!La6`8{`44^5N`6%@PW9frj@5v;jvQ_&V)7PO?b*rzvQiy?%1 zLnlr+|Lv8sG5$BAC&`ByzOLW- z_2TU**r!o1jzXgQ{Z6yhagSTI)?uqvJ8(LU+OgZ})Y|={9&Do%=p6L+HT`}HNfgk1 zDkjhPn!aBf@CWvlO){kW@39jSO8Tt+ij0DYp?%&Dx<7rt`}E^|fKEZBB9GrFjlvA3SJBK$4=1^DCyb`P@Ey>a~p>SVd!aByvH_uN<6U4~Bf* zg#-#0KqEM%Zpc^>!lO zX&obQh{w;Q!hxv1v-4VjJr(ki@+8#<^BewRAgWzU998H4T*+DN37il|@2AkhenKwk z0#bahCR@Eq$*X!!H{H&dV%!(0D7Co1m%~Z>RDg|Sa_hkp;nHjespPCfZ zyJ_x{+WkxgIj8ntHs~iGjK2?qpcX^wWqhO2XfzHF4#dBWMkDiYv)Ssj{?a@+JnA$$ z&332rmqxSGY_pNz8bCeld%-}5JXx9LYZTP&gM(IR8?X^ z5&0lAlJdCe6XmO}(j^sY9nubsh6{EzF>H}26#59mOHOTqg1XY4rLs)iwCZ>J$T#vr z)}Z1Gey_8?S2xv8v8pwF@=Vk`r!zU!mg|Q1IAH#0ddMsIMx{@g;q>C7KK50{ClSD! zOiSW~DwV!;8oykn+7sKu`)r6qfeUX;FPXV;-&8X1?nUGfFoRh~dF9y9XY z8Y4C2yRlN(n9NL;%O$!&^AYQwE53%(ddvZ^Ji6>l_BB+6DOnKrSFyJupFDXaqZi%w ziHl(_hw3FW47(0o7zpQew0sTV?eZQ>#Bu4xhv4v zXrg!I8yI2|(PGX4&7w1S&s&Io;XXkH$%34w^o_NOu2Z=CUmy z0I8CR86UZ6)?1yw@9e}Gps}C3T@{|{;1W4eh}k_5dM#o67~{g>5Z^%oqx)37wjU6; zrvIo(54Zh2!}O>N!^P2rX6GLLPIa)7e=({bai?__%|)gqdDW{QWxRjPA-i|z=0)#bU& z^##*uNw2`Jh>TTwm)O58Hn_xt+~ey=>%%;xLt%e5jRIn!^g?oteBt1Qijw@%NSe?X z=&L9y(S}3>D!-`bNxLcuufgtB+9B|&ni+!fPpflE#k30bMm#_aP!>7Y(iVcH>sjK| z>JXm_#n=$V2Qto<2BM9LA=*uiScdU(RQzi|{KN@k%vxqds>-W|Em@3MyXfVXS8N6( zG8~|XhS2LL+;8MEl(~e>y;XN9otIg%hT0_rYsx@O5{jjg-N)k04~uMMah&S@UM2}? z>;qRi8F6wrnRYfcJFy2)GVSsF#JV7HOcLuU<*Z#iN(qQXM*P$Rmkj!@2o%gnioPsv zA(UT#kAmi5{jlDUH3u5?ro47iZ`9k;&i;Wg>>yzlR$djSJK6}nQv+AZfLu2s6DfkB zGmLW=awbTgVo)uW++)%!OFt=3awsf#E)Jdp9;QszUx-QPliT{v&OdT^Yo{jt8twzJ zqhBf9De^7iq{OM9&TB02yz*0dhbiNjCZ9ei9Z8Z;i+LsXWVL7oz`e=?!GhP6i-XJ} zB>g2gt$%SGNMoNgW@Py8i*+Vei^wrer5iz#8AJ!K8#d4|@{q{(f$D_?4XzR%h2T2r zinh~grA{?=*yl?YiY!Qiu?b)y9t@;6XCh5~m1~3H5vqev?bbo_a4hcC|0>53@xpiF z50%cjcHBBXK5ibh4)-=nYx;D43Jw>2>>;|JO5C?E{$Kxt5`T_wjMe`(8%Kvn&9wfv z)oe7|R{#4LKMy+qRJvX42N9!{Q}AhL2Y@eCc~$4pNncMQoB53gVG^nE{aQRGt+Nl> z&lgmB5%$QcIK%wOYk^OyZTwsmySjTpwb$;3{^A|B84d6fX9I z9^7qIL95xUf>x(f1Rg7LbF8M{gH$F2k%T@O;$4UtEn+oh2(j`kz3E8<%j90gs99OAq5 zK_6WI`toxn^2M(Ixl%P7^dO}#|MR)>x9r8wm1-=>!~^kPpDW)!wHx%yp2*y=n|(1w zZ#88Sb&i`eF;^#t^1t+)d3c2RGHW&(1v4ovci=)$BQ=L~NN&XvG+(0(iCBzwBR$`6 z@DFs)mv$vPp3Gx{FTR-9F!Rul;<%sd4|0nZm1sq;C-Lu>9ksG5$i*zPP$~l-vA1sJ z6!?+n$txiYh93#>7!LFRD_E!0;I!(r589oR!_L8RyCv>=*uPHtUJ0O&JQ_2vUdyXS6o2(Ul+C>MUS=z4llB%L={JV_sf z?wB$6K;7a^#}~g<$2S(rV6btGwUWkit7ru8UPA`Y3HF8PG-kiTsUdE>Ct}y*TZu1p zQkB#*ANiLv#0v;zeat7MQpsNYk@)Nq|0$a6H2#-yU&MB2-VxVIUlGH94}&pe=VvDV z9`9e8KumVL@?rlhbA!Y$jG-M3V0i6eKf_CX?0IQn&~8>ivw2(v&Epf!&&}f$%lG+r za)OjMY&IiXY4M%mq`j9F3}t*IHs4poi$;E$uF4=)PBtyb#ooBf#W@ZkL*>F;JGVU; z*8<{Zr15g(b!+rxZ;z9*(MSo`)TYPrLzkiuBbt6hbNr8xpRm)Qj;|@y7viasSrtmw z9*kKM@%BbD)aF7}iBmO9aG^3$O7}_Q*%11CX(6BGdAY8-AzV0HW@|S z9nUB-s`KTPJfe4(iEJVs{Ghfgcr%dCswHX#b$ZSE#7ECWQXftysgGM$Qd>!FCG})U z&1E#a8Fa}F3dP~MTvp2)Dn)5mtrkrD- zbdl0)PJg{wqN+ZVd=}xE*p&;M6y^0W6bmN(70&W0fxzWgdN8FD?Iz*f}34WWx8 z-fky-JF*ctN1qkNkzr6ZdWWkmGx|Lz_j@+(w^iXDCV%slbagiP+ z*E7B;2grRV<#1-x+Q>u!qng43S%~g~^SomECT-8Ywu@D3Y0ZbCDfWAr0YypY;6>-rO~^ua)umyNNP*s6f<{Fb!f>vUA#rdd*`Pwx$fV?yKx7tWW9&z-$7=ji;QQvPVgIK z@B^eMw^*eh$~`V6QSNdcjF+F`l79ZMo?e=^{2~sy&(FC)1_jGhTz`jd;$H8VjZVOP zDX9-?08@sEOgYD?^WE`Dqu*#%s@dm8kTNNgzU%dyy+$MdND@A3K9*Zi8-g{{4UPqy zjg^4SoI6GyZAwsiv_YV$mqtPR~$gD2RW*Io^0yzlZq>@c6RXJE9(DVe> zz?4%c!O(Wi;TYZT@+R;yy>rblc|U9d4E`Xy2;T1hA_|4mC|s>xw=o~!eI|O{HR(&O zR(BQ%oPthPt2-6;o=dAc343S#+zYU`_}0;5D|IcxPA*!6y$HgFH-l+I+e5?`g3q)v zu6NJSBm1N8fsF;)f>uvak$HB5L?!k2qUYdO-t$eg%s(NNCOB&gaN4t1RxS^zZ2$yo zZz2S1W>XuD)qiGGpGPN88<)xbGP^G&z$~&~XL_ zwGJ{sC}#&g%;O;Jib!59<#KEzK~ZO%ybj<1sjeRxVs(yC1&2$NENl}nC9fU^0;PT zE_t99qLEzwpa1;lKWp#b*IvH{KmU9>8l7f`Sfp%KQ_dI)#r)wS+5{n@D7;C&l~<%t z=wl{VP5x$yN&=W0Ib|XC2kGw$kDYkinYd5I6Rd&=>i*pGf6Z%Wd&3A zF|87rlB0&$b&;8kAMMQO8zsq;va_Zb?Wg7Lg=u!$$&7P)b@vccMg}1qb^jef zpJ5N}HtL+3Q^!?x^`;yv!KH5RLA~8w_eN9wRqq=;u-%LN*(`R%^T=qA4 z7AMBTE_VX+&~gR5`{R@*j9nWrETcM?_xaQ2%rn3A&aKlho#!E zLY^C^X=!9fad^^fCC-TXPET`Ud}ivGDfB7(^%wigJ(XwmDf6Gp$NK+M>UGgwfJ4+( z?p?BW{B_hrp^q4%9193Jo(BZ8N_Q#PhlH{{Py?do-qNee6i9OZ!7(XF2G^+TAx4q! zg!cjKIw)k4CbFJ-vGgx-GaCyDGNS(#3K{NW2QmcS@bBH133!eN1Sz_3FbL5AW=rE0 zi1i)yZQf=l>L2#J{7e=P=0_Y+9^CX%1 zXyU78@&A8EC=_1cK~dig!Z%Y_2k0)dGn#@@cO)y*Q?5^*PCZ|#zcAaGK;yHlVKz33JcbK=yYzhdMK2lCRX9oFlU?c1 zH!*4V5Yk~46rE)D=A7b!;AI8}Vjk)L=ss4`cXNZ`*NPuj3rLRExDiqGQ8rYF6rbp- zGHZ$QQ|T;=D|@0{=^A?|_r^{wKhcwvYqni`8^8AETXkz%Z_}!G((5uHu3eSmI=I?x)f=|7eYp-#f# zZ$JD$U?0Q>!1SJ9iG+U92!X2HBuBKJZ+}_NPY8Wyh{D>4_?VFp`-A;H_7L4yJW&(h z-&gEd8$#dpus^5;C=|zY-$C_1DDlUAqtR$I4i65*zl}yC^Y3BfsB`p}=E31nr_pJ) zJDtBYTAhP~gTH{rlEygpC!!37e`zfIt?b~oV86+olI_wb`x&=Bq@voBMLQfy!K*!JM{kJgv zWM$wZCiF`uhs^_hErh}FV;r)lNGNR)i4L`b8Bz+XQ_$=jx0(lsox{U+TiHW$wu(vs zeU#eI)4^4R89Pg&*30A8OC#5(MK&4oeIyKeLUjljyhc9glMsjvOW4bzfU*$55fI@l zAq9*861LQ^ChdbcoPr?rOqjYk z`dn0crpBECS|74|(z}>d3VTu>%$^HpZ)NWkOD_$1 zztCqNfJVJj<^R%Xr2K34#B~6Ke}eoKfVq8%bORu^(ZBo;Fbxg*0a4sm{;|jZ+57D0 z%t59890j%nqr7}Pb$KtV)MA;Pb|+I1ZwpCxQck-cmPfmjC%W)XPD-hFmZDz#d+MB7 zW{WIyQ&D#=(4gpCUzU5oJnBd}tp%<=Lx5~3eo-cZ@@xZzI-^|10qnc@2D=d~QK%GW z7AVXuFebY|d47Si3V)%2bsU(8GJchqCioZ0Dza)}>JU@@i zJFxbC7jjZQ4?%e*f-$)WlIDxmj++X;he$$i8X%tm;)77+o)(-vL@}5K{g8~rl?)zV zAIi>LS`cO*SeUJ@TbRAYFk86;KN8$2P0~3542^gA*;1JPxTJq_IDK=;w{1ZES1qAq z;cYRx#ps27witaAG5Wf-)yMP0==Nk7JuX%2Bf;$QVy6~^KN}35fRzBU;gZyf4vGB5 zz{)IGWo8`x@BsK=v##@-k7;y``5RGVp3mAjWJ8N-R*Px!1?AU{4=k?PUR;yWJ^n~S zfS@y!TN3FBg`!&pm{Bl7E{1-hY%D#>tX=2009ZPTalR%K+NRi^o^f>!6T+ybNz4(_BYW#{fLW^@YRi1-5lJukUd-y|(m zoGX8JWYtX7JG9VhxzGwg3jRbPu@Gyi5G$pP0`n$kOB7HgSV;ml23e$TeLARhzSLiE z6eaM}-X7QokJlOXO$*bCU|O6%O4M)@c-n%q#{p+*UlPe%HVW=onD%5~TA2on4UyP^ zN#8u7Ne%eMDZm!OX)TM>R)*6oK3gX~E7NHiNjkG=$U>oMyqbiD${{|DM#!~j$f~L= z2Ff(Q*%)ZudZjoy>^yj7ifCMmS1evpw>QAQ=~3O9s>u_vhRIMcRZH!3pm#5DRaL$B zcz#ge;X(Ix>mX4^J7j&M;&EBuD2s}+&0AC?FO{fq7Jx-3Cl(c1RFsf%lTlHOiG(d> z2InMQVFkyUP@kz(ZRHNY8Fi2nR;i08T^;NFN7~y2$C>Gh{KT#iXe_ z+k#2+n!0vten@&a8Isz39`XT;Dwm5YC!;xiNgMaOo-7w{8VToscYAXH%ZXiC3~O#G z!h{m3Oqm#CQ}ETVX6_~IAfP78LMv)G8~{P*S8} zY+bE!r3*`{exzq1O}1$uvT*6^V;^gezS-(x6c?>1o3b(GKgp?7)jJH_}rSJPusRgdzLFC~>d#?ekM5nO$S!d2YPpWU|msGisgP(rvUH?M= z6-%I0(U6^7{fE-}AL_=w<3*IAQ2$+1f7g8CB0913hxi#^|3ei1$khKhIA|Q%`X7(+ z!&$wsj>nkVorM)VrmWvlR=uNGzdcjsj%n+5lvnM@mQE@t%aIi2(BL&&l0!&dsY+@I z?Nz4OMz+*OyxC^7cS(OA9^~=XLEpadet*|KIyrJXxuT$jO1?4`Ux|Wmsygf#dG0Gw zMlRh^7xiJ}u^c9MBFcm~diYj&q9Q5POm0|8nb=5OC{iVs*MUDWzIL0b3tvCq7V1M&P7hVADlAqMCL=9U5*DinCnyNVDETUZ7Y(qV z9HS~n^`?=kwM+$Aq5v#c|BY4tm8$-V6@TN@e$!XrZh{(sH-lX)v}_MuQB1bmefgiy z6)sb|P7uW!G%BuS9>=-XVZB9GVALbqD+>_c|_N+Fip4L|z0$_Q! z!CA(}Y|-ls=yl_Op9QEEpz8bUS9LWHmjLw+EY!15Z#rF93=KwTM8f;-do((`f!KpR z4|RpxF-Hp;9}wCNh5+k86u%?x$@0K6CN1|B*7ms+mVSBcssk<7QTKutNrWCY5nhepDW+S zAQ_{BGLUfb4jS#u*}>;vI(&`kwFO1Mf+7owEGXItBiuZo=+J^93yP*Q!pU?79CHYi z5F+SGlpr093jB#U!Vqs2!i*^zSdwk$0zr2r*18NyZbQr%@)x2CZv#CJS$^lAn%%ay zcF^^9{}&6kRs^v@q*Az^t4FY#y9-WO0_oS+n9SAF*2!a-`%Se1p0u z3>7Fa6g(dJdV|=o{^F}koNE-;UwrL*(D#v7fAQ6U9_Ih_!upG;e7-q2i94YIijP<~ zfI_C}zPMeH3RHK;$If5>^L_mVrjWs){^D!>#c$^y#J}ehMJ^G)vWRyJ`-A$6?^o!5 zej&RE-iRKq82Dx9NYiOkrb+Z;lV?d7U{~T{mY*aRLZ6N>+Xzk~5$lo2-(*husOU8l zE4^A4E4d*F);&ovAGw%1FmyFXZPq+#UV0r8E`z3+N>#xS!t#@ zdb6}S+7P)!k3XYeb^%{eA7R&iF&NjW*kXXMOeFVCcQoRo`md?wCR9?tlxnPGK&2F~J? zi-idn$K>A3Cf!P9GEC<&)Eh=RiDB|uCV}DD^o3^f!c^+QRN}%)(@33Yrcq4gR5TM7 znt24xWQB36!t_XW3P82`;`(ed+aA~luVlVGu=|0yh)U_All>LunP44c-^Dl?u!TE7YE z0>>0m+G*M!g&@))#`J?0>+CgOw4C9@mb7bF^l?Hbtb-fHR&<4P6j0< zQep+k_7)mUvve@)%BQ#3aLOmgrp!}c+IqOSaY)M|!smquHzN+nX2P>jYg;nmEhYEe zQ`RHdEO*Pn23D11frAAO@!vLi9k96JxhJpNymWO>8p_zrbm5NDk;OJA&qMQ^(Caa*-@w-vC^`1zS3kK3gA*(5o%f($4tn3{3o$L4T*UK$LW!_8_io|Fc|W?-`#44Z*% zY4~HMV8bR^TPn?GRWED;zO&XTYn^)P*v{rrvw75>NFFtLza)R!OqtW3#?Qq3Z+#CT zw%?0<*F&=;eKUT>=YMM+H4d}+-Ld(&*z8}*dSap3_GFW5BqIxWq_;-NhGMb(kMNG;d%YptU$YKW8%+YQ! z&T1C1R`%LN)@nUN)>@n!@*m5d^k2(w0Ko}k6E$rQ-BwAjr$i^8u$9v417S~UU6F`& zNxw_)eP=}!`z2z4^ugu5?>rHjp)l;iAn-79S46WnVF=FT7oG^sM#RPa)e-EyYW|5( zD?q-B{lSWebuL?fBDA`Nn5~!{e}|Zr@Z%&jZgKdsaQJZn4sRVydBi;)hr`uzhg%%JYMNObK0D1U4!1abHMH7Z9M13vN$K-JO`5oDHhS$}5R*T9QiweWgr`DR`rVAdi&c=H-|YMueg~~;@zF(owiubO zpK925yvRjo!Y4QFnUtgIq697Cm;a<0=35o;zm`E}^{a5I`lc4M>CaK-(UJA(FkjX# z5)o6;2np-`$cGK~LCWx}CPR>WavRT^OcL~s(9|@XQ|97@d=r|*@e77 z5?-@<=+*F-^#?J39+|QZU*7x9^lEA=)FNNffNSzu~` zsRgFnKt2mhEihdJ(T94xZyS6e8Iz+I2|eFAzLic<#uyt~R~L%}I8lJ7GaU z3H2W7@G+kKI+p#Gz#z{S3M^?GdoIosWrXtyv-uMipx+2ny1LNq_p#KUMPGLd-Tc3xhoYY=|1L)f{&MIJ74KdjeP% z9qkEVPQuv}fIR`&6To(o&z=D63BaBJ4AM+@0Q-jvU+J$A2#%ceHGjc2oC%VrC(*>gKCE=suP8g#oX`f(8SYKWZcxBe`! zNxTByX2JWUecbNgo1r*uAB@|>X*~k~jMm~oZ z{F6&eo{mUO#c(U~JipNGrTokm|2+sv6wDk1J|X`5;NYl{iT`f3J2wCKqx_7E|Ne>3 zwhaQ`92Kw)0?*=0u^}zDCVXMjwnF@ND->)X`0Cis)~Q?u6(8FNYtjy6_nwVVX4+tI zyZ5Y|ircjJSpSn85p6*DqcG8K85illgm2ee=Z{+H`NJo^Qurcib+RvE zDjmVHlGIm)pIDE{#fHCc$eG0Y;Xf9eJZlZ;vi7y-<``Q@JsS?c1R~m)_=PEG1LGH_ zp$+g}oQ5{Tz@ik~;y8JGRImYG8^%rcsIW|G+M~iE)U!tgdsNs23fiNBJt}Mf4ee3k z86OpFWbqWyujOvPHn?~R!Ns$ARX^4EVl(2gD9E@b$n=y#ji-(OB|Zy@=b^9`Mm{I% z3{k)n;(r^5t!5_vw|!vae;?WseZ z?SbAs*wu_v1p#rVIcAx>1|A*);?B%-7pn6Za)}1s@DYGl{Fn#=#DCMneKyXQo6d^O zvEE+KB+hj*FK!bP3eR$oVZ~?7gd2@6TVsu=&mg>$gUEkJC_-Hq1#GxfFu5=@aqRd% zymR0+@0^mki$Xvz%nh-5%esldKuJ5R&RIJ@9gS7zG#gK! zp=QzN2o_sE%v@ZZa~={02{(6zHv1tNfkfL{1DysNf^(Um1bbEhZVJO5k9UW82L3R! zD04Q1VHelsOlH^<%dloAHiu!yDsr07z~5#TP0pq;?81tiDH!&|GOW>wjbYf6v09ww zb8^Z__%cL{ufipG;;c!Nb0G&XlQw6^Mk7t(mt`z~Cl`LCiS`HSEsI2f)%v}0U{EUzPX|eR^q%R;M- z(5Tqjq83_NXthx!v(U;yD+{f*f^gdht*rCnm|WlEjVd;c`T}XxXSY{9xfJSYr?Hf$ zqoU4J%A5XFekS^V4#^1VKWgesSmPI`_XQpA|9RAD9&~c~-w&GB|MM|^_?A$CzE7Bl zG)m6kvH0Wrh}a1#$i)oy1lRYWN5$S*8Tg33b&YVAtiPx@mdo>Y{D_DBYjKDcf;sC; z8gEl60a5u|fy$#)g;A!ys5lX2V)J`QpoSqOD&z)*A$AefV^2L$sHzZZ3d21kr=rNJ zCsH0}N!w{(&SZx*_Gh){-y+W;BP4fy)j&)&2RcLa5swgwSSHM<;t-F7KS1Yt5R%Au z&6!N@|HdU-Mn1|rX~jn+9Q5FBr%?sXqgE9hbgH0HKRGgGn0%zwKB|Id>!b<}8vL2# zlY(dX>zxDnU!#5!+caYsm{|_zd01eorDAbuF5A?IL}Ke41s;1r^y3MY}c z@t(NiTMqGk=dChSfYy?P>i;95J$d#alMpkN-Ib#~q7+t*^J(cgZakcSJ;&xcs?N9T6A3 z{9M7Vmqa`6?SU73<)tSf+3iFF>ZqQ2#Ng*jvr%&MN9Zs?#AfPti24GHR5Ei<84{r@ zjWIJHZ+V=0mBUUI7+D|9G!v#^wkwq!GF^f_xN|_I(?C$PzFd^`Rux#B^hDHYB zcyQ4TcF_Q2nPXjF%;IEqP--u>-`@7kezkWKa{p-tV{qK7v-{4rNcmNdtxzYg`?JqQdwLq^CVj*@? z%LY6WBzukNwFP8z1G0i!7LlzGk@>_&tAJyrXlr$NENg+CWPm*x2gh8%we{{WWWH+EVRKnURT_95NGl=veSz!Gi@4%LESxi-Cuh z1rNG03mz^ zkgtEPRDn`W@P@VzHCXKgu&}|x2DKJ905)WuQ)h$?Z3`Q8V-_}8*f29}SdLGpR7yOy zZ|hvHp|Y#1^p)KbKqMVRtrEqeA&Z8!|4QNuqxdxt$_SA|vJ!=U6a)Gj-vo z5W1KX&JfS#vhy$_J11rGYPRyS6T2?*FQo@|mX@-jYNM0%Q6>sb_XOSy3N|pk0`hl8 z@UG}8_6x3-L=X#J$v-XgzLsu=d{3nXf*##sSFWyc;y))4QzAvjPs2eT5a-Gl9Mzlk zjwuK~4$)X~gOIsCFtdz-u6W_%--}cl}3+4pZQjU%x;D zm2oxm!sQV6b8t5k8^X7MA)Sl!x_=hi=1o8zUDCa9;|jYk(1iEu8td-r;GRIcn>ZRdH3*c+z~04j6xKv zNiPq64z1*Zhdna*0aBC?HWs2L-S1e|%^>%XOnRi!Cj-Tno0$xfsn0F5Bb5C7=B>`Y~IXMBiQ^_6+f0$Sw=z zgvF)XjZ0PTxGhN?SD1Kh`%}jiWsX}J=9M$PlnGBGPlat4o`p=S1+*!JtP5k(scE-7 z2d_dRE^oa(HZ@;u2*OwHXBOYjiSG@ct{k$p*nh5GJ|6bpz=Cpp;<+dkd&Cr&yO~Wo zoeKXeG{+z3K{tXf_6IpYDe$%dw(&@m@ozWAfy=lms^Kw(9Eq({ty2U;XPCRnU?{hl zEKnQdwl=+kK|1Ua7`im?$-<9iqHy81On>Q2A5WU-T;ONHOzRU=g3L*fF7qhIKzt@= ztn%G=;H*a?1}0b%mjH#3k?_mzkgM5EJSbwK3zvWR5RWahlfcYbjWBF`Iz1B50N*ji4oF$79zAig4o|z7Yf_nGpWL# zW)S;w?7U~j@Z+!4IgNj~2WqiZ#ea=tBTK;twcx5rBXq21q}?<;DyG zoG64W`DT^~Q`0;1-W~`-@&|I5=%!rvwKeogz#Fll$Iz( zCEv3J-^~p&Ru`rAe(lXzpyva?bZa~>C{Jz*33b#kWp@n2UPA`KWm-%b-q{60EVa#l z+KI0k1@#lLU07Y+C_COvT^*pi3^X)EZ`_d-221J(r>c+_*C0<fgu3Re zO!|u1@S)kT`IXjz`G?^&f8?6|&Gtp}Q$pAjv30TcUPx}KQmrI>`7=@HE?whY@03F_ zLbay6brUmW#}<9x;A3-)o3DZp|kyqSF_DpIHCBeRy=hBc!Q)MG{VK!6cQRXl`DTX0mn|}&JWuhfnj|*HbYI5ba#>X-&w$j(P z+OSBY(XQ~V=fbq>BX&!|>#pPR#oQf)fzu6E@={dNvVGt-<7Ayl6HeW|Q$#U8Z#q1DZ1 zoap=OSN7!m(&v%$2UgBoIX|78SNe?b28CV9V91cGbQwBdiCEqI{#d$ zif6b?dilB1!!GVhf0lv}IvK2Ier*VS*IUik=0zjoL*P=hN>`#+nvdM?5TxkQ$UXDd z<|ViI5HN|pU7D;jhv6Hf*NcUN3o5bFh-u^Sm&J(|C+hp_$B9{^;L#i~v14JPg^9Bo z1r?kKLJ|-PJ=%51XoOh|WL9VRI#-_ruk;sJZ2HR$6$XKaA90{(@_VzRqQ!>UhuC|> zQT`}CH{Bp#%{Iv5@de0tu|HV$IL>9W(+%YHhG5tW*k?cw!D3j8VK<_m7Q;RV44XA+ z&VykOErzuib_NVPK>?@KU6BvN`;~feU7WuH7m&?i`J5KpO;G-&M=F(l4sp?6Uw*Fq z@{g5|or69ZGBzy@Hbv{FXU<}!1+h}21S^r%daNykl&15!vnWb8V^P%OTa4#{QAZX= zSr|1LMh!7#WDvqp_umop8TQa_qkeKy1!;|vNPKh)y>6f0`%c%5LJ{y3=jNBfX(GU= z0xy>vbp~VZ**hSu#p-YPncD{E+u;0cC!hYoySg^4vr#ekHul{@C3WjrL#2ta?+Zq|+wk_M zEB9@9`<(r>;(d#C88&L&O6aE%wZ7sw^`%6nZ+a;Dii6LW7jC}c$`~78Zg$wl=4?hk zEl}HHpk~9yEl%5-u<_*uimyL1d|9#JYY+ZjQn>eH#CI<%qT7aWTbQ^p@XNx)EKIa$ z&<0>zG`KBjaPa`_MMJJPJjQxiQPmr++_GWRHjH|Lca%jk7RlHkXbWXF49aX}5cD&S zdfsp#WW$`74vB1-^Cp8H8|G}ooS#CNb9s1kF0lE))|sE@XQue?>tkAT$S5E_@)@1F z68wbt@5a$#GZX)PaM-r>-yh}2#(%Fe{#*XO#r5Bxi*mz8id)^rW1DGUfR$Y%ME{Nu zWmA>|r~-$$^QVS=c;K&p`?gi}K+n$Kf=fXxa3P6hEgP$}Y;4F8*&gL>8d!bbRyDVg z-SK4$Z8jC!+>-G6-w}#X*F^yvKE(P0odd=pJ`0KGp|CcBJ{+L%bEOKn=zICO;-bJK z_ao%9Tn$Y5%>1T`xMlE~cWfn90Tvtl4e_E8qRSrnZ?f50hCQRN^J2peQQ%<*Qnd6r zzmP3oqM7D=MhEG|2CriDmOb)I+3YOCe!YfaL&qV}@_Aj2{!F$$3lZKtCJ~VDiameX z!+)dNon_!ZKIXt(h(^R;0mHu*4bRB$KRxQgXON+O1i+-*vG+2oa^-M6J-3H-eBj>V}UTVY_<}+7+Q@fG+k2 zIo-knS_RSIN{j{zjPn*^w$yR{E*o5lm0*!^hRYsjIKwz+Gzd5dd%M_oLIf$AW7I0V zS&tn$4hk45tDCSSf}E+Zti)EZWQwRT@YNFJXZexY%?9f{BfOPF1%~?YRYFF4SXf<| z3431Hkn@5KeX^lX+aCJ#kEn-2A2Gytavg3nw&Y_`WX|N`kBi0sOkD|beEd)A=`j5|5V`am7tj227!g`hx%(LOPD;vIJ!*@2gpE=gB z`S$g78Dji14_hSDrAfM$auW|kf& zvyp)@OALqE%=njOM!jrmn9Fj5TQ)qJWhF5zHfRMr;6GDNtm=(2%W+1z9xu zV;jRg77XyP5gp(eLv9<F-;`LG%*~D{7j7h5TPV-1PK4D<{}2M zH+A^Og!qq6NmCGzq7oU^O*^XLD4-{DgA+)pxdd8-(`Pggh& z$Q2b|xRxPbzwhh<9sm6Nedm$>yTLvd(J0Y+XH25|aK_}t$Ud&BE>?nbGF zoyB-FW@o40X^rD{OX}{NFKJJdv}#PN#;B)xlZNXoA%Vf6IgQ0t8Cx9zKb-=2IX>rW zSST>tTBD&;%21?z|0F$^F}R^XJq&npeUO=vHcf$LGls zHT%_?=Ty3?-XSQv{8CX;EXx(QIN-fEY`S>wFm$#_U){WkFaxihO0xv?kdaLa0NF=AIF<*JsC@FhA&X{z;v+|3k{PnU>I zyeFRm;$NWDd-7=%!Z=2u_vDigL(KmfrrwiTKB0N)1y^GXZCaIy5j#(y5OkWvfYV3} zl`;1J{@=&m6HFn4iTC7__vAk(Z^geS6lDPsf3l2Ii{r8P z#`A8=QHo-Ak5;A`%7w+}cXz}!^j6Lb@Gga;5%yaT(RFR~wH~6@L)3bR`r#;rs=uVV{4A&&r{iP*Hr>-+Zndg#))BrxA z4+%*bO3#NG4uf+XkBEVMs`>W5^Enq^JN4bWH?MnZcrt@y(TDVZG~J!z*M<*jpk!;U z4;e-8q@BE8u@F4*aQ}dHA3G z-eC#jB%m2dstmY#m_Nc&hMv;XQW&w`iZg@8BS8a#=iN zkb)NAvp{frVD5|XP3(S2N`jTK1^8?MK3ip`1^8?MK3jm#7T~j0V(+y6*XpTXmY#Yy z4*azMpREFTjovJy2HY0LXAJ=@%x4SpdHecD>qKjnlU6w~zgzrH(CQ{Fe&@1qmk$B6 z^FgaiEktLL;8t1s5|yP^Cu$)!Tb<~R>qPelVsrI_R4~}|fw#Mkq`7Ke3G~c6e!i>) z%xn*rwd#}}&Q_gj0WVv9>h9=M+jk1_@~2Zk;_V%9iF`)V?cUA~_$S!Di@=vvO4mI& zFTwTJ*Zk&;=*?qwtuv@Mih{8DF`ec7TOv-aVMuQa^>T#)y(RF=&5dXY19O4MyY-go z4MQZZvqo*UH=_4Kp57!B)HMv| zEHT=GUw&EMo)-MF_4Yg{Z%+$e*?N0g@XA}%Sgut)X)!8qd6brQ#oPrQvfV9gcMBi7 z{@6lMwosHU6lDuVDRbNw6s7!Fhd{YO=*jlE1J7Li&jh7H{>DDC|8@g3%*JD`vd)DA zor(Y1@AnV(dnNqO!@d2Z7XR}mo@>JYENT-~L0Zrz6rUMOywG|@;x5333U@-eaUqT{ z6P}7*NzG*b%Ov{>x`2|79UMZDZJ@w+B9&;_-x?)vm63`}Edb zo4ILFP|H!u-f3S&(UQxFSjH>*)skM(Lc{KXQ?!wf&sqh>I0k2Os&y9fTrJ}>YaqM74B5+&QJJ*!Tw(>C<@Nv?mQ)-9MEw*CK~q@!x1v!z zTScR|y)DsJqfnpL)hO=WKI?6&6e*IW?t%!h2?=(APXeO~P=+qiIZW_*ivG+HWl96c zqK`V`-=Dwl&}_)p{V)e9Ehrf))giQtIkl{zcIbjXe+C~vO0vYCoU)r%L$q`8_~f-w zMNVa3ETD*h?K9mm6#Nun#!%2|C-QTxn1A}LNyN?q8Q&TgI$DJUnQ{>urnI{{FJ^b$@RK`8p>efDgqneW$`cD#6D~ zO5X_%(5on6HBl_$+y94B(hBWXXx|&5tv&WQ*aeMV``jp(CwY`oEsul`zh7~FBm{T;+13V^-&-U{)>#pl6Zqc{Bqh4{f< z>p8!5A-=Td{BRY|`5dVK&06-YjBjQ9-InpQgX%7x(eorFAExJ&krY+--ssm^e+DAuwcXT8E=;+!uELwHq+SG}oRW=IRK>vHM zFSJJoH?J=EAz!GQ)EDYXe8>vJ3(?+IW7urWe!p?c;MRlo`|Uo${nHtqu7WqvZ=?3_ zz0%M+85YnQDi8p2!@!6B1O?}k^Q|GcFK{d+wZh2~e~(j=CE~Ay%ki5P-!DYVz)zxO zldtw8<~ z1Tv_T$h(#tHWC`geu^Ni0o-d4)xYkE!}06L>k=KGKYMorfzOiOxr=uVR* ztsK4wa`>xNqUiQo7j`R$zXUnFwqg8n0;#-s`uY0~ukL;4^ZhlJ#YI+i#_|{O|P=w8aZ3M0(>1oDq^ND(Db(AzrGcZi2V?HmVxje#W2PI=SIJESt~uR|(BA7E9JNBb zQE2O_R)4Yooi)4nR*5FtYt8Ofa^G9Y9biUMJyt&dcQcY52Eq9i8dBn4Aa<^j02gf6 zrcO~rE>TSel7j6Xcmfu$V=W*0Tm=Hgcp|n@FfB-NE{=-jWd_ee3{+< z>e>L>mQfw{de^pX->MF`r8?}d5&*Y<*s2b#>hJ}t4)S8P85)6~i@0AOouBjc=USq# zbFHklA5yoX^`16RVrycx>wI zgqSjGpgz_KtwN_!6e6FASz3}SqJ|Hmr%viL0X=6*3F?MPTz6KI**NyYEI`L$ok%mg zNFws)O!7fnqGln(HThx~sV+0z)B2jruYBbL(D^e%>9kqlNzsTilj5A{q-cyjlvrJd zUf|^CG(4NeehqPY<<@9U;$Ob*KFg8CQ7#%fqm)EK7?Ufve;BC>JB+B8okG3@`xip` zeazqeMTD5@s99uqox0rO1f^nC#6AKq(J&j2`J57Il{jLfbo6r%K2`JZ(#%!SC7mmT zF3uPfT&>e_KL70tc~aSdZbV`(iE%vcj&O)*SAFOvBrreRgaocnVg_-HQtJIdNxVQE zyi{7!<1;EK0(AQfo$F zpTxp=6pG$>2)BE?V6Wfb1$&2wyP$6?aE5gr=gk5C4;*vtzQ7K1E=DIq0@I+0iE-_; zL~UmrBPP^0Zdk}ST#b2Wv?n1Ulxg-%{>@MdRUBFWsQ+y5=y0!pu&;z+)uJ|e?-sOBqfZ6C^)CcCviZfnVYZwzdwK9Ir-`3`_3+Bg@XF$|9#(i zWPZKx>?R}-?-CNc@9cs?-uIo)H8%2&IJ0IFu52^E`0fmtCV=e%-`ZIqrmn$en$vM~ zN*Rha*JCSm82ITF80xNY9FQw|UZgUkaEuI*7buNU$c^6{rgYVzJrkY0;Fx~9_LvrX z>s}1$Z4kNYXi6xB#Rl`m1_l$Q_iHRU#rJ4&{!CH2;Ap;F)g$?xH1M@f4CZ($BXl;w zBUa6!(W*~=20t?#;=iQcyq1&lx&xbsnSUWgyWN->-b-btlu~cprx_q0M=-1sR&XKu zNmd{JT*=eqS(K`1CAri^E3Y?quJ{({tgMN7{?gsj`ybJzA-Z66&e=9?&App6_nvLQ z+`GM9+1A{Gb@Hq!&0IpHrkx9&gmd!7b?|ZyFqPZvNIs=lUsG z*)^&tTWWlkba*eic!HGh)p~ap(X|n@?qYf=3A-Mb_PpT|C=_>S3hPc$xDv}dhBv`t z?pmjqb-h%tyt*dFd+RIv<|tZwz?NIv-`x?9z+2hT;9UwwBkZ^CfNOFGBuVQGXq^GA zGeG~Yy?=BCU%>4F%eB6M2j&Y99^2bDVy=md`|Jm}<&n8=I&RGea2FKM*WI65rSqDU z&Tm_#vsF4AXEm#8&CNXG&*DwHeX#s}yrBVg+5|CS!Gf;Z)wp*@zK6 zPZ136e2<(WxTtgQuIN}VF>MvBR>6As3YNit$N>gS2^CBT32A&om=Me>e>4r9;@5@`s@Emu)`yIucXFoJs)cuHC=wDZI|#Yz zP}Ceg?}CH`_itFlzYF1+OcsVSaqQ+BUCEY(ox^;wcltf+$G-ieALV}^T7UQLANxJ~ zi=_`m;3t%c7t! z)-xObW&i2Zy~7gzOaJI;uf>14iKoSXx$pQdt;4c~d}$q)56oeC+mJ6C!FYLCm@ao0 z#AQ9ymIsB>awlO~*1}Vhp)N2?3ocU|zt zp*q&W=D43&9P5B@+;PRI1>0z~qAx-#YQZ*Iu#FaMqXpY&!8T+SUliCzBfiGEpc-@W zAFfDx5fW&DK-|-(leoRQv;?|lM zEewPu(OYNO{c(nEuYd#5>?AT}%t2`7`r;0F0!S?W1Wk%&xqRTK2r?9$52xoBI1acb z%Kt5W;#v47tu(yveBKDQ1lU%5Zv*JKO&wngGqIT-Uk6R1k%EUX$P$xc?&vNNm)tbE zbwCieG`Mpre+c7h?#h@stV={YHxFwC*5Z~%)u2~A=z;y2qIB-KwrCi6O%G|$phd&D z^VX-%*YlLqctOM(u`aH6=PwoXj#aMW_9Pj={C#Qn766cwAdLN*KAtsj8?H} z6`K}9qt$Hgm}avf42{KnJsXD4cnJQU%EiPR^}=rovSTyDyV+Xr$M{xnH{Pfte!~bM z*RzZ_Qu+ND?h*>g#y#|#0VH`yJuM+XmU5YIgmZp5JzYXuncLM%;I1quq;8anT~F=z zY{VC3V=c_fm*uu;VP0CNP3yD)E!s=#v$=yln}v?OZr0S>K$Z7HHM(Vg%?%FiChNQ( zW4ng0<}N6`?Wyhd)OM@z$`7r=+d|s33hzTvc-Oxbl<>dypC0vF{I8pM zTKumT|Et|OkiT2}uXg7^f4n6y{EcFO-6eaP~B>j7*ews>84 zelKyiHT&*>+1KKAwPxQAGd5sBr9agKqN{pzGF$ws`7ugO2nTo9m8^bP1emgAVtH2g|i~CrhxX1sviVW0hY} zCzl|(8hUvNAg<*FQVYg)05iSLTj-Wm+7rhv<=M zA-L|8H|8#=x-BR}3(C-{x$;A+=C;UNt(yCg)ZB-IzO}MeBH*{K@3hSQ1>RaieTPks z%)6r9wRl^tYWGE`b}imki?`L{ZMAq?E#B5$pHnT~R*ScF+Y_(F+q%zqTQWft(AK@c z*t*M4BmP&Gj8hn(F7;EKFxm}Jh!`?GPu(lbCN2pBSMWIM(MeMsv+=)r{r=&83IFS; z_jIqt|GJ51>+jtmj=MuhCyqlA11^&Pd~lpC5R(A8=tDwMhEj*i8lQzgP6ZG^6p>gY z4gpM(=egnF7BcRpC`6E==hevuTo;dHk|H-I%pH*|4xS4$(%Ax5aWH^M9R(-}$uvSS zqct?BOe@yTt8yuW47m|vDfa37+C82kjS)lC!=#I2%3$o*)x29xEyPr{R^Ylmq&^JL zbCM<#7&j7KB2Sbu%0i0n2z|Fb)!V1$sp0?r1g4C3t3%y=!M}J@7==sKF^~Tr9UhhV z|6Z?u*z*6Icr570fdc?c86H8OO|l_h^C9+8Owj-wC(xfDchBq9C5kWT0DKZ!o(nji z7yvO|04z=W}c4!Yej7TJ9gbuE_dlPJn!%%7o4Nkm!%{hJ;)(AJi2pFRnrH~;2;{b5sO%d?Jj53tEA-Y5%(A7=|r>kZxbCFTU zKb#9Ry&`Eq2ac=C;(wSvaa>F2J1)peb0Ub9jVLms2;y)67)9v-G}f}?*$S%@eZ zF;0mC)I=0X`p>@V<+&mlXDIwvnxV0WGLtgmFY;6Yyu`u3dLIJci;zha5El53{JUm6 zq`Om)xP0$$19HISbclViBpow*554}8cW@0NbSeHXcUP7c0CVtvd;Odc`{hAZdZ~U?+zL0dO;_C`77Fj#u0Bl(um%tRoO!qk>K!-<}$qOcg zGET5AusEb!;PnV_gO?Gx@eJ4mQ*edDP+5a@k`+<_!I`qg&JYvgl})$-&7iX-vkwt~ z86y#7*oR>_<+D5uPynEx5=z0>o^(liDH7s=eghH`P>@k>$MVtyDz}ScXG@e6VCwVs zO~D93mZdAm>n%kQ3S2=Am(W|_9d9zGd@i962}&`yfMKYs(XHcur3l1?8=`9Bk29FU zm?1OphT#jz%0Q7msznH*yjAXYNyPMBp$k2ESt|ll3c9ERI08>$>=WKx# zjq(mC8nyxPY1b#Qj}k^LynCO-EG1!xQrgYy>~85wMPG1oB>%(|bWTYa;&}XX5SY;B6KSQOZjv zVE`tWa;DA+OA*)0_+L=0GTvoF=wCRwXt6p#NEzN&Y8mx=PKquu*K&T4W&VgG%m(~} zvju*{l<_)Y5MZtm!4UW%1~46G((%Fw@I}BRpL#XYV@q@q`T(OOL<8Wua-O;4zi{HB z*e86l>;l&f$=Ds?5bXlQeBK$nfB1Au2&H%!^65OLq7#P*2)72{vwQ_Lco>plH-cQR zR+BHpu4$4N1rGYaQh1z>B^PrjN-wc51X4m$HUQ6h&w7Hf=^p0YAT{;ZC!)oC1OVkg z8o#b!8Wcqj7p6kVe4!I_#g+u!H$67lyGf!8OrRGu^kPx(sL{Mk^gYO?K7@bR8)zx{?I(!jO6e6FIRMp_<-#URI#$x}5j0w_fSY`*50Lrx_zF7Hs zobU?77%Dwm>HvZ$%ls9=k$`g`3P{Fu)*<=esPaYA43Ff}N`>_2d~*nWDK}q_l#o)3 z{{IOD9+;fr`mveebIxICAm z8>~i0^aXIiX%>r~=AHQ>%os{x5aAdkDZZp2Mu)?Lx|hd@K|sa|0s9Z7-sml4353>evQ08eQpvIpQe2sk&7Fs0%W3%;OV zh^c+52ktR|3I0AMSwcAn=OuKl(mZfBAz2u3UL`2PG3S$iWUa z(7A%lpNP+b8J=?6UUI`N@S@1^`IP6&=)=?lzi>rB=@NS8Q$XTMQeZZng^ZT&jsX1d z?%fGMaUjB>!M4%i_^6=}@!wNENmLa(p3(wR^k;@BqeAK7(;Wo@wmutAXU7BHrXpI;a{Cf>SJmj77SGUv?TLKVMPpw?e0bT+q34ZEE=&t*ObIzvtS(YU%6B6v03V4C&1Czy_&e2mD zq!&UpmEG(nByi14?&>1NEA)XYtBAR5g52^KcSKSb=2^KQ(0`v53DYwT23!d^C4_y4 zLnJLZ3E5%DTw#1%D|PIslm zMG?7_CX}D0;S>y0D0*nbPzoOBUH;fJ#YUK^f!YGVPcp8T_agm5!pjc8(ZL&RS}NIE zIa4Li{@xp`R@@~Cvk1N6ssUBpj)I8)n07_C;uMLd>m^rE+=RDjf}|Rv^u|)LQi=1ls>D|gRo)1t zA!4dDMrhO-y(bV@o`z8thEgq*A=Ab?E{Z-yBb1`p7wQfdyd(v}#UveVLQH6_M40Vd z>Ikcri#0KhbFKB+Tu`|6l!q_Hmt5!94xa7|Pf@wQEK@{w+RqdLAByd;U(Ud*f1iQZ z$8UtzDrJKf?@Gkx&|$T!=NU%W`({&d9BT4F!O;lcjV8^X5!hN@gAXL59jqQAy)?j4u3FJps2b)q^5syiRUKJdpjMU_Z`XP#!b)h)xXHh2^#0`=77Dno{;cCQ6uhdOZ zD?GU+a>W!BWWK`@A{on-axP|mP74d-7bQ9vQ7c;*4|_HjQp;b-CYkC7#*)3l_C=kS zgMXNin524T=INz5bc#eXxK3=AZCdkAoWwyk-CS{$K10am8%#wTE>J8f*De#aenc*r z>K#wSa~Ifx;NQHwNDH{d3igDkl=4V;0jMrg$|fA=He2oufYf&6X#Ohoec=)l#>sHX zQ<>S9PmrGyE=bjgSgn$+u#RwVDNRGt=c?@p`Q-vl2Y~-Kf6YQF?P24h(iRLJ0SHb= z0JK66q;c1<#|I*=Jmy-5vw%5b+~$cWxyuEyF*VdXXKTwzNT7*O24;-_)5EIBPe@=V z(VsU#(iTh-m_~YeY^YF*d~W4!w#+H@S!lD6EuiZ0bYc&N$pn_d^@KTwf|HbRbDA?q zIslKKJYJ>@WldRS48UW#y{o_J@^pvh1r8BgsJxWx`4P$sbZkB3KT_>io*&!wqcCSc zGHXsdDpfw|Dw@z1D5UX5A-p6ha9!}7{7Y%mrTT1b`i1aCNcH!L6}2LWQu%N^_AZ`@ zZn*4s-`Z8Gd_E3w7Kw$<^U0qg{a*Kl_)*R!Ngdd6GSUpndr4S(vfxq!L}$SS3ujeo zM1dei=Fm1APkEf+Gm+4!dxj^ufRR@*D;V{1hqhRS^dg~giYV1SMCGE#aot9}qWRCEQ`=Tc0>Itd}AZ`J0U2nE4n`%4_0C?(O+D~ql2G>cs;2+>~k!LZ8-e2HnA z3HI`B7L1khD)wcf2)Yy{P`ZplwPRq#m=2NNYVmX|iAcq{REjC0JU#jVeWr3r1)MXt zE=hqn;s+AQ7)oQRH(vxN5XEpF7@&A6f20UR8Ivw!^uZ4^iZ7Ap$bye`rhHdvHucKY z%v%tfxRTp!!Aq5FUIQ$lU`1lco$3oTEqN03cA?zWmOkOf)Y;N3Qr7K(Z*y-uPpaCg z_&EiH8_L27AS_{HMXA$bjtGuZXJ>5+<@a4~_FiKC$uvU~qxw^lh*OY+Sqg>EHxhN_ zZPY}|R!&Iv`U9yL{qwy|Up6XGt+J>wYZGI@Ppi)Xhmylg@e}XWsE8#G!C=`6QIO zRCA#6mNkKP#pW?I*{yBC6j@;cYHN*Tg2Su0LC?u+<2s{T*iORZSXeD`yGwo+f--`M z`k9jW4>FWL%wb9OGe0{he(Q(>`D;Q_{1>0^VL}2C)#$@e?s2)=RzmWZsX&AWsXuXq zSt_Ee#P29$A~5F)4JU+L6f`!OBA<)$r?eD>p@{W65~#ltlu}F?ikaL-^FxTE`cLXD z2?C=r428yEKFB|Gu&4f@e)IHJ6sGzlhw#sgFeqQCjV1NB@Xo0p))7ecf~0WFDK;G_ ze7SSFM>YmB2K|W~GiM9D;hy+sG)`b`-wIO3?!R4UE7uNm^Z2wC%>IMW2*hNmE67$->_+5DwrGqMGFpm7H(8vm ztvr9ev%<5N!&|mBsC`&bvr5MIi3}UH-u*h8=rA}TvGdqDo zlMoEb3e{AejpAO2jujPAoE|UmJ3CU2cdFcmR%bi&Io0?22i{kF!p%q|7olQ#6ot7( zTM%`{5QaF=;Sk`B+DWWEDwYRD77J6FsPaw*D3ABNH~WpuKF zn5_|pI@<{ej!mYz^DZ)UMsWbCwkIT(dy2el6^J9bvPzbZ8(#%v_y_Wtm&a=6aYQIe z*c5DwFjHooSxK!51p_okfTdKl>)BD~wzTFVeH`Ib#Q0bRd1?Wf7qnz+4wM{stu(G> zWE7)KbEYR)Ij5^>D8vC5sV;hYuF9y?Boi}QE?gDrD3r?J)QU|0trDPx{Yh>=sabDD zXPHt4;8ZaT5g$NO^ZPgs#2&v2Ai2qz#o0CBczz8yUZw^axi8KW<$G_%?u~GaBN$qNE!qrFJs%_K z8XY14LwqUM!GBJUfjl22!XF@Gd~%DafG)AGg4G23jqr4cSr z9it}q(J|#FMap#;KAmQipm#;BasqkWOPra(BN3!0O7*~r4s2F4jOXh7G#YhwLFaXh z8HOSMQ=i7_Ncq>{{G_8pHD$eT2xs{G1Ls_YxJq8AaKNZTD}}I189+g$$kG}O*I>^O z(t-#KQAn!`ab@;{f?E9muFgK?9%03(|oLNj<0dyOthR;c_JXh%Zr$sBi`jh0-3X zHz)iB@7EE;VV0tI8p3-3_8lNLP`*bhES~@P0YOX5<#A8^n*&6HFbrR!5Khmyr614% zICOwIuxVcPxqd=35lUoL$>$G~F=1A2xgiDvTx>$BU)_f`+&1<>`z(q7q2s-2_w7&b zUY&XD16!nyIrvX|{r$ZX{>T1duf>15k*6jM5K{mF#X`brdr;`!p>XY*lfTx!&+*A? zpyG(QYND+OYAj7S6u$L|f~C_jRnlakp7U;E_Jwonv~t4BS^uc@>8sV>9OvUl7vPcUgu|}ce%OQkwwY#y5`v47%@Qt?{yRh z;B$+@dq18P<^TBvg$Y;BSu!6AY_|OGAM_8OmgN7zLBH3^|C@NWfP&l>OIP`;J2KMp zT)7F)_360|a6f-8b5?~|rN9$w0kKTBB*ifsfzCf@x3f|KoVS$!E>ND&;}X>X@MMqp z6gFSL;5pdVp(4$V_I9ii@Tgpf{3gG$6h~*-XoNq2PPbE^QqV5&DXfg8&M1IS+4<_) zzXg7gK>FZJI-_})be}l-tAPw-$pe)RByYBD+NANw+*q;;Tf7wn~>H<;qho9UG3b;{fR$8>qbe1saWz&jz6L zmYgX9_-mpL+6+MF)t}a!3k)h+?3v-;J=n|zbWCS(neBJl@2X3zeycbqvvBwo* zSulT3An`3KF>;K8^x$6~H7BjqsdbkStxKXS@#HAPR=ahZ#kHlBi&$h^C7xHl@NQHU#yQ z#T0DIqg&wR+cTb4#84#2-|?J$A72(?V$nnosToJGxYfzI?W`P$7-I zDIpl^gKealnC}$}ktFWOaq~nr+`d3FiiRTiM}b5y%Egq~ToCB?)GkDA)Qae)RP}P* zo}#iLbELUF_bmRx8MuxtfC^?xF)W!%>>Mn{XGDfhrA*9&Xcx#K0Et7WmT$`XU^5{g`DucQ171dyFzdF^^ViAnt7zOTdDoQd# zrUM{qGx;P+NQ`1O067?UIZ*;r=91BT0(1LXGJ_fGS}Q6)s@}*mjr4FG8yN6<+6}BU zZ*3*ZC-7%RSYb(5d6bQ`CqDy5{uf5!j1)~}+1Pgcv-({ejJG8&Q~Xm-t+Nt{-(ozu z*qwFkxOhmuzPa_O6i28X1-XCq@#DPyvbI=O@-iEN4pREKWVG5`KOw2E; zjPhX0s=_}s6jtP$@+x9lfv@qv>p01TuXH=WJ_8*&BaOUPXxCPj4)8w8kUZ>H%KN;t z(7LIXsmEPJo`Y?laMirlu(h&*%9|Zvi)QpE*yeW4Pb03zbRwAiDuY^o6F@N|E{i^X z7R%NJ)~Zy}2oxo=NM+WY2MUsS-T=ItAlX?O4znTTYvJC@yX;ZDN21o5;qBD$C{dnA zC(!zq7h3hc{DThuLI1_2eV(dVR*ozPwvE22OSrW3gj{m$l!{<*Tz+Lys0XBxvNAol zM5Z+zEQ%ND-|O$(1-kNv`>|2(RD^Zoax&XA^(!gg#o6>CswsU#ORBEqwR*g+z`{J9 z)0Tx7Up?h_zImHnv^Uqjr55xZ8=FOz{%vn}v`^_-asN*O#MtKqr_I*_&5i%*mG}Ri z_70EQ{lA-d6kCLGko&g^a7g-O*f#pJA%y}=>c~0K*^#zt9sz3u896Enwg%3~j)W}< zX8Y*2P}xhj)&Wqp$L&0yK#b!6#S9!&RDL_;R#iO191o4t@vS_*vrY$z@d>k_5p zqkaS|kF1uhGH|5|T~&dtG8y%RvCyh`0VgEYc*m{?{<4aa*L5@nV_BXypdJxy<&$s# z-n}@n$F1Ox(x;!y>NF+PR+*r-+S7w|C@O1B@L5=a$T)F&8=%yZG%(k z5c$Qn1${++eVHHKwr1YCjN%@0gL#~wkr8Mw550cxPJZRCt($602D{BMVRNSkFt_vC zsMl7>#oJ~rXGHlL-Kxv3{>s<_)zt*`qt;Bl)N5&#=$+=8>Tq?TCA!ps zzF)&E3;>D~b;Zg0%=V@gnqEX2&XdiVx20@N6`2+Spr^1nYi?gzY7Ln6J};+gXsl(N z)cVp2sN+?CS{W~16hQcFYA6D9R2d>RQg&S~gPS|4JUA7YXOpo8-r4HN#*C%<lAPpvA34y`NZbjQve62M#`n82TQbQ(Ol7r`fp)`i+6nxeDst2|k9H*A!ve&dX znJcbVWknBYvAVup-C|DDMeBAK6m)TU!kU?}0LzjOiZ64Hq3b>Q>E-#`<2SF&2X&t2 zyOcx&>m`sUB$O+p@?Ju`Xg1|pchgj>Iy*UjaSioJj6J(^3UdsI>p_bw(_UES%1==4 z<7$f83s{xSYqwvzz;*c~eg5%dacY+{+GbY{Yg;bLSVPaPMQ)}41pXj62A$o`T*^u; zrWO~vT)j*raiFPdV9IS`>dR@St2A7-RMmk8H;`tDt6B8bmbfO7b2@DnH5=+DjZWD> zfkwBi5@0eeRsSOkS#$6llxzOjaGZjc6Uh+KN3kfy)L)bD6QD_mSsi&cEzh~_!s)eN zZnGp5bv)WxplP#vy8`37G%<6dyp4&n2(@@rD7`fL7PL5LLWTZ}GRl6A@t-SCyec$m zbM=J@$cJNlMkzT#JK)iBG|dx~LUJtUCNlCA&$}dH z-HT_mE3$Qsr)!zYDjG+*KBK(2T*ZFg-+L;sF#8ROjN~dRJ>}7`_Rwnxt5u$UAk~>t zB5vNMb-9T7UDT|`HBz!*4=l?;W_VW1v>13ynbx%(q2hA%=MrPD>MiYB?!pCVBWb=M zt21{e|2TtqE0qlmSYToW%)nKbPJX9XX`z4u`e1QQS&2%QGhf5V+2E!PPV7308^&@I zq!$n%YmY4zQq3p)2MORiO4_b|#D^oke(2?fj{d(aeKPm8yq2OAc2B>rg*cyg;iv3c0*; z(jt8Yh{1*~srCA(>;kyCDy|yZ14=lCmGnyGJl_S6BwiI)J?q=0DEF-L%2iqByeiHd zg9e31fmT@Q5B0n}x ziJH{K;!1dtjRh6b6^p3COE#8LPFgIej7Zs7OfjKdNEL5WKQq@+G1~)LNTr6|xs|lR zCDhQ=Rn}0cs5KT*siaj`QK^`fmr<#l=51cPVXbNOx*9zu&ta69ehmBoj z6blh*0qHi8_zO`+pueGZ0v?@u9ZkFl0f^88Z->nkr>Q?HQH{7%1dd;aGpo{t~ZhE5^+)m&=G7p?2Mjt<|_d_YBLwEz(J{aJDS=ThI5y2C6E zFmL@I?jMxb|Ng;#yZ&$DnID8zhIudtf2GB?{;A)B26^k}`s6XSpAz_IhDxV!2jFqZ z?WMX(E&=fQ^W%GmVzA=+Ul93CKJ)P3j(U~zzek7d`QICPK5h>GEypI87;utH6hYo> zMxsi=@z}zNX;I)-M1dM+T*I>NO1YlDI%=F zJE&;2wkh_szF}st3qKMP6j2Fhj=z<9;W=REsmz1wOt_YF^OZ((%HB;AWTwz-Iq%f3 zSMtZ5@QbV{|C3<20Pu%;=Hb8fdu9Be{lnf~EB|le`B=dOc!_D6C0t~Fn+0RU6u!DR za#pk)yhktQb~$0afrq!7|3ICPHErS1&^h3>+u4#2La~w!tg({_6`@x*hqK@ zKuFWfU%ozhQG9Xo`epti&EjMFGet_0rdfyv;PIP`vQz%$Es0MFVULB#{Sn96heA4$ z<~W{;?^%*?!zgAjRCnyx7Wr7dh4DDw#NPtnk#vXyz?&S8<#m{ZFfir?c_uP6!tns) zM|C3jVE~Q}5BCop07g`z)81&M`UEQPr0;Nu|*_tIB_xxE2QZTKm8K z{ipr1{O|W#`F|tNeV6~sfsS7r+Bjbi%OZ|vp^TPw!6Tmfow(pqU&30rgnNO@=56O* zp``MCq}5yPzX5%(f9UnxUZ;){ac82&M5hDr>ti`gkAExReU{Ro4-Ca(Xmz?AyHQ69 zi=v8l1V%hP0GB8o(t(gezdp_=5-)X5fx;ti-&h?D{k9k09eFmx|7YI;F^~W69aipt zIyg9N`TtEkUl#wr?n@_D5d-UAJh4XY`JE8r;-mUoAvSO?#71t0Xu0<0pB4GP6)!#i z0*JZ#f4>_4(Qo6wZsb`oIRCnDekjJ@-}?5;jmDl7gUs?XrUUTlpQ7l3AkbbNtqdPC zmf~T?kacitt^U*dN`(Q}uXU{hjm=H-6L;A#|GKnyb@ihTt$0}HS(X3Kb^2J(Z2sTh zJK8Vv|NW;e|G$~%V}7B-Hs|WEDd^A;3w~fut7@=mbI0=t;t+IRSWqDT?3dYgKOPz2# z(?==8BkXflhR3mp3j8xel<`JXr+wAERVYD?Qs=jVb4wUh%@yT7u3tNR_H=eZ=L#}^ z()mrFs>$PSB+**iUYN*QicBdhP>_=N4>DZ1{-G+ZN`xwCc@hGMA`(-y_(Ch*lXYf`DP)cMxiDFnhyYWcT%vS3V^I}LUY;_+UrEeh zn2B2PXEI1z5k=IoG@`S2r<;8oiIISx2^>x&mI+iDIBdlOGHXyWy#hEoB zX@Fx}q}Gul3tfR29APH6+}BxaRx!)d5M|ubjK^1KI3eUha{0_4JuEn}g9uYTeNr^W z;_|Lp9VS8sW>vd)-B_oq){mU40yR{>^_Qz~&4%4D&BW`w~tDY8pio)W6TnpP)46|qX)(!ID|<;F$1 zZ0+h&(`6=Y(=@fqT7?pl!ZBx%%P4{>*G$=6EO~j#E@6m++fX{@RJ=k<1iOcfL4UH8 z8MjC`R|;R4J_+;rXg)Y})jpY^m?>T~lQ>2$P@JMmjIIiFNf%n@x4NRs-l>M!B{!C# z01ls3L)EGGXaR_&+VzKXxiOQrA~sTh1RC7@sbkSuAX7M}1Z>J_7M7!)=cKY)ZV&v; z9j#fz|35qE$9m@N{~lD%{~hk_xA;Fd@{}Cuvpnf#KYOibyX;LjPV`4rFZxdTdcUPS z;<<-=Q&Bw|@?Y2bqgM>`F)n&Ml3N)Smw0h6V?D1sDtOf8DezhKl~0wJV4}uQqSXV5 zYP)vPt?fs*%Vh8Hvm*c35C{ux0ng$8`~8D5|3Ba&q2>QK@H8MhiP3!!C#h>gbxK#Q zlL)a1%IKM&!o*-j9R4f2OvJe8oLTD*(8)J;ei#nXKf%BoX? zT*AMA&!0Pv1+VT_FTJQjeJq28QWr<1lgA_|+_t2D*OpyOT~j&JyTL18}<@hLFJETNuLFN9pSmwYm%M>L+R? zUuOnNd|jq9VdI^OZIx=Z_E!y0r0zHKaZtF~Y_Y=CnG~73+)vGvH{V~Z4q5R2c}rp| zn4_lMCT%5aUDw*FJH;b9w2IlY8bHoL8h5PY;h;{qH88 zdKRiIFYN%B0RdXD@2=}u%^C+!);xC7+%?N0HO$D~h;~rD=A%@~Dz|>r-S=@5H+{?@ zeWw9os5REF^;wVqt6XdG|9-!JSdRZb>h)Xxe_Im4M{nq*c(5nQ$M+QJw#2n&mKT0;#ry;Ql@tQk9Y4)oz7bm<~&42&hn zvT$Rs0VdXjF&A-$&0Ws(2NuT+rI#?wiPUcj&YT5u&h}T<^{kk#@8ku7dHsdg^N6S| zV`)mnR;^hI#vd3<_VUWDm-FhklCDa^gv5HrTAx!d|F|d3PS_7dCB0Gz#s}vXZ+%26^c2`1Sl;T~K>| z{+l7$5-tPnHao|Gb?P+NNSIt((8|M#Dk?tk4EpIiQa1J5n+|GMKE4?_&BamHg4UC@HLw$HN9iv0f)r40Ke zh?^!GVBYzU-a!Tbd%wm1zM029|2GGn+rmA{`Fdw3zlu~(iW3qX=aa&qf0v|1ZE;FW z{HLy@2N6VKU)Ni3)6{ofihL49C=O6cwKku>sSQM!6@j2l0STurv->1w5XUGzQNT3j zn6&=zcG(*RbAQnSO*CK`fj;SNc{OW3F7A-3-G3{1>*i6Uq8e++kzWcfyJZVojkpMY z5O*0Pdn?zxvpc8k=$u>L2D>oAag|E*ZUrjy^5#=1cSDsXR)jLQ=jK;zU2HQ!BM)(5?BwPFY zjt72hiAS#z=ZQCj7tRxpMTzs|zs?r;4@@!1 zD0uzy6?MEMCEU<H(yZRoExiQyQU+V+kE}yJO5I*-#t?$iu{ICex1RI9prb zd&IygN(kk%Zwg+=BjSAhch?0BePAJunHylrbur+&-#EYj{<}|NNd?6bOYj3@k9LQ zoG0q;2q1f2AvdlA;%DN{Sm6OYFSSvso1%%lQ&gYpAt<*RtIxllzWVXi@!6~Mx5sZ@ z{Z`utH60Ir9ffdAf3urbKv4T_d#mt!2Mke2uDtU?M<^oZ_Jv9}r#pD7e~$6&c4u*Y z%VkT1Xi8B;E)fMh5udRCHdBySosbyCi~>k0@iAmt`0$kSS^_~E3PlESPc9Ouz+R#0 zhK{+7Hz^lselkv>5V%dv7YWb?hutoAep_U^`7PN3Z}2!30DWMTk_b@7XLlVVlVfzHW=dXsY4jKYm-}AN>p716SNn-*9jm?E^Lo30A|&~% zKY{TWg=D-uL+uVjGVCh4-94|@eeuKb+wWif_|x}Z6v$DF;2(VKCK9oL#G+D|7~oQu z;($-~cN3&LJW%b8$(2YwBHRw=4OPVq9OnsmnUW+yfmRJgWny++?tyLj$37q_;0A9B z#6tZu!zl`OoF~A-SiC?}a0U5i-tpo}5wm^mJjva6bqOupXFeh?Y#!z~V#2==$ba}R zJ)B~tfp5`>YrTr=;&|M!7I4f`(iM~8T&#;NKQGT^6C|gSI~3))*p#>9fp`zMTS!7&Bihh?O$%@%LT8IKP&2>k5aVy6r&d|2&fn z7R{ZI!Wb~5Z0q&&|A4_K+3}or#9WqzfW)5M1r{DAlB--_!U47~F$56&Ufd=7T})r} z0Y*uPcI41N@VfUv`7DXI>Ln<5zKkP{gylA~aPP6x<6lepASZ|l=p+j_EWYd*VlN*RjkGir57GF!bEs#CuVe*%*v|Jn2U zT-^&0^;0b5`2ZYCBU;Kg&e1NO+436bawmjWJX6Y~#2||-;Ue;uU(ip}8`s-g2qJ$wZ&!&(4F07 ziigch*4VRVG9P$*Uhg5xpQZS}R6Caz0JHgj|FD0!SLXlwhi&}djXYc6r9yU4HihC< zH9XMs9C=z)2?`b>sW2uq859m#1p|o&EnZ6Yt7;No!ptdL zABt1u=G>@O3^-fh=QI>$2^-IrcKM$JQIyy&2bh}u_P_xkOG7Di3*KQ*ft+~s(swkb z=U2V2!~}tCBKuEYJ2L=%P6(IleQ&QXu0W+=M1?J;?n0$V3jdhIASHwquJqdiDH_d@ z{q~=j(hSk=)85h3ubhMgXUNaQY$7p3AH+c|Rr>(+Ib-GIl@b9mlHN{jR!b^8Lq!qV zMYuf}q!+ZSy%xw#NZ^|MU8*V&uh0kP&?;Fl49V3=iZ5}9#^@FGVJPpiHf+MTBF`i* z2~v`XYaPN+-Qoa)pW<+ON(lQ7he)y&bBr?0;$!-j#Qbyd<1>g)jm#iae3_wY)c&PjVOs zsw}b8(hThaC=0?Aj8TkI;cm}8sXD^NtV^v2%Jvl6Xmcr9_0JRmA5t^`$G@C`SN}c( zuaDmdb|fWOiEJvlmA%~f)b`*N^1qTo-=2ZOZi>DoIS8cJvdZ4%5uqT~hgUUI6#N{emn!gYOE_x! z-k!Jb9f;o)!PK81>uXH=oSA7{2R9&G(f5@9#UzWSUJz4H=J#ke8sQJ(qw-E#1qz`R z2SjzLa2;_DI{=-%PAM4*V%p;j(|Dq^*iimAU9O*GVueQPcZ4D#-TQmb-eB=gT#mS4 zdMEe&A`s$B6e|%iM9$VMMwg-yN>S`1+eCl~?p~4fA|xITgd01>OXOq}!u=#yEq!~A*60Mpa0;Mzv)sEM)CGYFPi2hH8_*g#vcrFl3&d;C! zxAT9LMCe>980TX7J?bc^^(|1j;YUtO%~uBC>#sc>J^IZzCWU4;I=zTg;ght}vEFhy z--)K@l3YbXo=`yaR?)z!M2U{q6Ld zZ+09pi{A)71)$;>gQe6zqUaTyup&tj3DDo4 zcW@A*&aVDJt{`USCuInROOjidGx{^p(|k}Mk5g8Bljk3Zuf^oXN!Xl6Cn=;zNYI=m zAwhLEXe1`1Wf$bs#%2$^9G~;PIp>_$nntpob9E){IYsPzIAw@l(>z*^OfjDng>O*w zj?X4V-U0*x;4WVF@g>9|9END|^1qOoc!lGDTn)f}WbBDSjUKDDfrEp6bDziyeQ$s$ z%DQAgtD+8*hh(HL$hPqY;FGI3^hq`hu@7Dxi?~a`CXfM&$1zumCkPy$yw;b{0C0h( zcFq)Eay#P!O|_9yNUin^d?91X_0?w*b}|DfzQiesxpfXMVTw8VRE4lekm_9}ogztpa?)J22MIk9TZKKPQLf1{~7=CEZ=tpSLp^1JmxbYcQ2y&`E~&*((@X^eTf1&-q>5H}S9n2s~) ztPsDC^-rr=uFFE|CUQ{R3p924fIjac=66|0`TxBOG%cj&LsXN@Pg${geGjwv^&k18 z(xb|laTVO%mcUt}?1XE!rkzU9<^Oa^+~o<(H#x0e;#7F*raz@Ggdtm~gwjw@P3P6X z+e2w6cCKjGhwdhp4W(Ad(iV)o7_3nwEoEZ3i;{iPc~ncgyR)sa?g~ zmBv_Wix!%sy)#C0Uoc?1-F|CHJ>E z`Ja^n_-MYBx3ibsDR@+8q35m?%3#VBa~uP!6u>ng;i(YC{V&PDmB^vOo@DTgK4gSP zU`jHs(E}I?eU>31X+k&!`PK45?N3M`!sbM|$0x6W^ay}V2JkQfR3wmyb`chYjvJ9F zJ#fqm&{Idn;aCw_qGXGtUl6=RnhZc<@ao@Z9yk-hOq75irqXflRJ{Mr7Wid?iXD?T zdso!JtyDM+r{D@o?-E5!lEjD>HQ(WsvR^Qp{2&W*w(?pBjftEs>nIiwHw&ZjR4hJ6 z|H^O4Iwc}f>t`AD3qWfB))2%WevYA5DF$Hg@E`j9S(c8~8$Lr{n!DP+6?e7QOsbP= zDr;7$E)v;97n(kcaBQ?1AR_u~qI3~rWQKhDk8q3vuR1oMP!cDw+`PWy@a6sn#&C?% zL0u6(e1TH-QhBxp;9&n*uQyRLIDh@@=}~X;kT+59@w3$aZ+`y&?N9Gsoq6m7TcnP8 z=f95jO875(`$uj3&&@o=gQ)fUzV=~M@cFa2G<}FrtfVXFFRx<&!3bS(u97iC1#i#! znU+ls}26ecL89!u7~|Mlpw690E_ z*q;BqnP&@}s=zuv0j0hGI&yRMTsf1^Qx&z*F?T07f|%DvdzfrVieokcoqy17XQcw_ z?r-L5rf_Faa3+r*IL^n9-6xKIt9%qP%p3-Rxy*!{L~?^(#2^Ys4c&Ek5^>~7y|#=Q z&cyU?5Cy)m0{jUIvQXO{gd)2{=Si27X?L*r>hiCyRl4k7SK8_wJ4&+S&c_QBD< zx+UgJ?5~23+o^AqBmEBmnmV`uo&r9WA{Z9HBoPhqoL`VEU8HBc4&Ep(o_)xTH%g+y zAvn?e5k?`mBE|M-gqfbUn1b!$RCLYDw`V*p7s)h1{*LFo9u?0-DUX6a!XdZ%)lrTi z;+A6oAmv&?7KV!2MSD6|IBNKf{%AT_r)f;E&8-70L5VWK((dTp8{m6EJC{j zWTo~n6{6r~FFI>BbC#h3f%!}pGg>VVnMy>qEk3J7rcUD++EYyatr_VjeR~J!{Idf( z=baUJ%J*su84MxCz6#1)m~@S1_4%`-PsEAV2>hcQ25V-!JqZ9-zCA1w{)NCDP9>O< z9!ce!ug_%8^5z#aR1+mIWiA<&k*j7g_x#?^Ic=)m_GB?CIeN$Dh(LWCOpG~ibtTIu z@MlIC0yD3Twqqyd=Q5g7HANk|kDMar@~$N(N4YO*TWj#z=Jl*K5YiFSrJ2|Eye_fq@13UNW4jNus3=v`QGYbv$r_hB zvR>!<9xn|{0#=4 zut{Cl*#gEep(q@w-TK#*W+>g|iXBMB;oZe3iV;e?P2 z$&=%;ost#~-HAP8J>~54(~^L>;Gpweo1Z^(BY5P>hPhalTgD7sM1B=vLu-Zh@tk{G zYc8s(W}FT7*$h`YTghvJEajbZCi+I~n#w6*^h(Ib`GqykLSJS?jC5gGg^j@Fup0Zt zhS@55Gh@x?rr6MKhu9ioKSit#?{(=e4b1kR1(9mAm*VE3Y>!-X2Dx*y>)01?Z|c~U z4?Q*gylQbBLCT9`70pm&ezD>>GG-~G3MmK<$tAK*I+VB`BeHQoMqAuCXA?uyvb55C z_n|CnWAk24mG5v2!*IH5rh`6>QxtH95yAv(gbr;I-thOfEI7gjO&2b&=zXE+;`kx_RyY9@_&QomLeEfj={P7|7owp|M&Vw zE&j`mJT3pfe*Rxt7n|h#Yf{pRh%f!w1pja1`}&y||NZo7zkL7a;bDvaeVxX(8e_in+V1V=Xf1e(f&wm{D_WJGmzlle! z%$TsssRo6FUuFaOV%I<=8}U82Zf8e#pO0$HH4lyuNNuahx)j zFXUHAX9B(C*b?58&aMq3aq`p4&aN2?fh&=afd8wIV1O$3I~+2UQUfhh!H22fE7Zaz z%mYg(K_xT;C4d6h46vbQtmbZePj-sPCyz{en4{9v0nP~Yl-Z~=m7e_k>tXcU&W@>~ zib->mLuKILF?~x|pr?d=n{2bIo^z}@AH+G;nQW%IqggGj)Bjhs{|b=LZXUo= zXAjP?|Mm{|EB>GUVSE1nMxIrT3v9M)zeQVV?!h)u}l#x z0Fu7=I_lfxepE&f}0jwacVI?Bo>L~-~c!{I4krrINT>`SSlOh2D!;dX}+3K zfzY&&IH@gdp-8Df8_u3}pqaCcH?@IX3K8nb`E{IKygL7huy{DL5zy-Zp6a9xuj4eH zw$jzDTKxI7Sh>9^`H!;_t?erQ@;Q&<1MnPfU78zD$b?-^9>HyfJirQF- zKiPaRU09a}Yn)+7?7?V~d{xT@6}+$r!2e*nu%2iA^uK-sT&({;tlt0j4<7V?_wxLv z2=4>MZ6I`QMgwhZb1Cuf6Ud#UECckPBM zDN9r`UhK;vcy80SbR{OW4Q9SHhForunaFm5+g$02zu+q0*zg>vR&4yM(~UKoP#Ez; zk>bvATQe_JM>c{ostc!WtmPX1OsE(WP+5fIR}R{iLtH)fPFeizC_Fr^pRlV}K1r#^ zTTj+EW=S+IG8@&rTFO~mg2fXDN3{l>MSQ!%L|3itH4yxbNqEiki8kdbt(OfIgqY^m|Oi(!}vc96vpZm4X0*}z=&z*9Qr%qPaT8$#*>)aJZ(y9BMq|^RgS1o2E zXCwNkSEQviuY6w#DgBHx@yvm}NHbgQq`=uA3oo$x0L?TFJ#R24`bj?{})|0azoC29brMsf) zUq60?9nexn(!~n+=3>ksQ?MWFRKe~{Hs%tpwlgiwy-r13!xd0ul}p92>W&Rzb(Pqa zT}tWFMkt#LuXuQ=JVK>KZ^?*CnY?g`+br=mF=69&EoEYK>(^2=F4)97+pC^Eil#$JG*mPY7B~9JKrm_?>1*XS7y9aq@lGEcs4cuJfdt0HLscWeERd+ zY^~MYNkdBe(kWT_+RCBU^O+Y#TX%E=_zmqe!d*jnDPvX3awvsrU$i3JsPWAU!Y#3A zJyaSmSlDxwnXV{lIbZ#yj4p=L5m~00GFv~FK-OO9ylwqv*xhVy$Zcf~iEN;bho!2e z{{3dMXEW;1a;1slbV!(a6WSpghD$HSqLxJ|H?Y;va*Q{w|+&-T&rU zfG(o{^6$$1@99JSkNbJP2K|2^{0-!_H8xZW&)g6)s`I2%4g+PwKbd$?^)OJk-2YiL zSe_wV=>JoV|Gc|*_>lkiexAA-=6TFnp2;oreIAX7aOKSIo?qgQSxxm76hfNQdPfgo zq}H3gV2P=n#Ocu#$4PO!ci)s%c}rC{18T%%V-xtpk14LePF-ko za$EY(mx}zSo(1%OZ|`uYM*j~U;y>TZzx)l zdS!sw0coA@`=tY>T&p-Y%5~RuYWzW|eycfglPR%$cUl{ALf=RjdsYD z3mLlet(^F0cF%XoQ_C&06W{e`!2AD2M2oxa4fuv z)BHnAUp&vUgiJ^(a8fYj4Fw%*2VZN`;ZLLEcOtdE##wEd-{J1`mG3Ya&zi0+2nPIE zZQsWgkF>`bjf_1Hl!ys`?D(k3=iTF+6L@KA-Ys|0-7wxHt^6$@{;Np_W$zE(*ME@b zmi(WMY`qTqkNv~_n*GPq{RjTP+D<{uB{9}ngqcQXHYu=?nu2V;-l?z67* zzsYRGZ~}|u|9dt2kKMh)hxlLj@~ou)v+-sh>_5JY{m0ig`&il9N}Jr-811&Nx8(IoOdk0lNGfN^$YBYC&Pgib z5UM|9e0_MDo?Fs?rRH4H1Y{BY-`=k9f4lp85Anb6=V=%FwdQ?Pp#uK2^nK0thGLfK z1A(PP5I9do!fi509d4^t?n;=5hmy!^L9LFj)mHwR%pM5q!(*RY)Bo0NPxWUJ{Xf{P z=KtM4*naT;zn7`;KPV8dd5< zZ2(VxXj~1ZfEHq&oNAq?`PtZ7rSR|5{QHjdAFG?+f@5G2{oi?7mH!-WA3nr?zn7ph(Rvx*3xc4e~gF9Bvs7 zukncR!0RH&6UXxmxIx6nBuUUHp#$Yq7pJ2w#7KfgJS7O?Vi)gm8hKrmk`a&$w2?6~ zj6awhbAQ2@=W(LrI*Hzzd$AU!Dufh}T8ObTXkS`sw%_MKR-EFp5PF z{#D?6!QfZcgMZD7@u(;NF+celX*{q0lxO01Y(kTPyWm6 zqMtB}Y0lBf^B3F;GDd$Uq40t@BDkl@G5YiGk)*A(|Ic0=J%9Zom_)1fF@OK>>_6S9 z#Q)pf*?-9Yd_PYYo!WUuM>-FIcam~}lY}ENCg@$g#QP1n!1a5*Q7p#!APDKCSI97Z z9*>$9&?XvE25lazMAEO7791U&o&@Hd){G;}2+BE$P@JNF+NRN9hTgrj5oXXSjnI*e zFvAPL$`aLxhP)>*zvexbr#{XgmyIMuy|!>Qy-i6G;ZJl!*>ys3#L)mtkn)VCXhw5} z=uN62Mp>RDBtk<*CkUrD)|nkp#^^MTfTUd$2kn8Ps<7=)NGICsOe5bI*eI=DBjV+$ zY<8_+g!Cp@-b{O3;6Xxqn>dCpYq z1UvhGgEIdh0=*>31f7{I9WmkPBpp)EySlmxY07CrJP9abB%=sN5pvc?NGFpVB3YZ^ zHGK^9m}3e<_SKM`R`V>vg2)~{@A8}w^zO=jyed#9-vxZE76l+r{&Oiy_0$rTUE(xC z5e*^XfzrTU!dUg*P~(MkC-pJLy3D(?7q4C%oxiwzbM*Sf`^FJBY!i0?%5N}Nz7Ov= zx~1QnXh0HrBLP#T-Od+|SC?C2$Umm>5`)gu=3%Shz#3;{LZ^fySxByKXEDizPg7X; zm~$G&SQy?wmQ;%t03-=ikZ&1D2<9Xxjo-qt5CgX4GC0Ua3CMtjaA0xQi|*+7EVvZO=P1`6mE@CKHBSjsv0yZAx#@Fi(;hvI}gG z5G2No=A$v1!o3e&l@nQ9L33B7%oN&+Jx6%@p==;|y_MLZ$T)yS`kS>IE${yZnL zFqRuV9T7Q1X9|^YXbWI6TMP~*CRb%?=lTASvXIEJAj;%%D=1>b*P^WQ+E6Flu{+0} z2b)qcBk`|hE@8i;@5xZ!kMt1`r=ynr8mEHMUe4ombP0R68y4;PVocQ88a8a6*7YmR2Usl&A$rVnhUvC88w-kVJC}P2~w)(8|Xgy;Ca=q_gigf}Znpa|=<1 zSPs+7NoAR9vcmgy0>{Vn#;oL(j^~K!8q$?)R7@@*$(Z?WJ-Vy}YXtC%NZrW=LXO+A6oh@XB z9+s~`oP@9^^%TB)d3uCyaB{5=sx@ACe5cu7*SwYli>Cf0Pehz0j>C5v`Qv;US8HtVCuA1^OcJRz5t&;G0P zKRKO{O9lHSXcHcFK0j7bPpZkpY>rj*?YF_1{`%c_g+r^2xFW(a;eBP%E*~s^OlFq~ zCf%L-sL`fjt&-+i}P95*k_ z4SeZ2uHD&NX1OIMqchLPYRX74-Szo0tu)S<`S{zKGXvEZoHYa2^0Q`z7Yk;sb+XQ? z4M5{KN$3sAXoQYWe}vQeHJMO0t8`;zRQD#8!W`WY37@(HU&%H@ksv{{E5o`iw*U>| z8W*7Xs6!oOxmg{St+?Z(}-URDsb}A zmiWuFPG_rO1-;vwyx+9#ZaO*L8i85{Dp?r-DuXhi5&6rrP8=nqvt@qJ8ypM!Q{I`9 z6~Jsr%t)t%u{bK{YLpPUznwXOZ&=Np1gT`@uL6a2|6bNh#6M*QQ+sRgw0TLPW7!{iw!IE z{^<0?TnM+cP~2SnK?E|)(@@>MVlk71cl~1d03C6}^Kh)MQS$ngQO;wKO1a=q(141u z5p)53+7>NXT?4t4{tuEL%F8X2@T(NxE2INiX#aIsi~qcP_@Mv4m#1QlgqetkI5cXO zBo0Z+Ngo|$I2;pyH`s0}lwR{b`sC}Fi^^V03Czx6pXrUBv!st2HcF!$$<^?bY|nN* zoMrasPVh9?_9DVV76W70M@LFRujC_=C=~6Z(n|Km)Mk`!gQy22*{*%>n#^t}i@<-b z+?en5)wa()dxZF_p-9=^I?SwRTlLZ5e=^)Rvo<6W(Bi8 z(kr$%i5a7eM1Cs)L#T`18nzdhCFlkTJVG~P%83=h#oP^mH~lN`pg^%XNA|4I&INeX zM^km`AdsJ{Fw}i?GJHeDDI=VK7j+bm2p1qi@oFiV(YyCv*B+G}`0mTbeC1f%0orU} zhenKBN9m>4MQ`%SfG|Xdh$ayl$6R7b>Ad8r14=azWH6c}0~}s^#gsehcf~@-+-U7~ zwmn9su^jkI)#j^s5{th4;Hl`~AXkI{;flx4xJ|6Aqz+CBcIE4?`#rOBEQcUx>4Ks$ zNir}3z#@h~lQX3vpkhpzI?Oq`8OPxm1n4?Ic@?t(GsK6w^ra@;)?Y@QOd2AIU5c7Zya40wI zhgpFBH7*p!TS)0=C0(f$1SJxqf`m%DS=!^knG`xapifFQOt?^{4O$H7D)zNnTPXks zxhPivMPJ}aFwVX|IzE*^fJ9b-+DAj2a5?gBSKdJpNvAQRDJZH7qK7%ms5&iChGWK| zjj2uf-93imgbZtG5X+L@FR0c5q}=^+3*;(t~cC(r6P^Y%f%EXJl9mq*X#}@eBfC zDZZ(|F9)-?`}ki3w#LOyr~bp3yKngK`$Bu z&A<{o9>z51y76#KC7Ci9&pBb*k_p6GaVjY%CtyIDhJ+is&GHn{)TK_^P9TZpnXOy5 z;A|=36Xxi3oW^n$0!n8zM-fFgI4zCFsg|eO1_i(yQn@$cbfnu);#3yG(a$*-N_r(J zBMAxTn9aZ~K$`_>?KewiXuvRFXed|J;{xT61KaE*HfbulN;`pewzntdg_cXB!~NH> zr^cx$ygn**g^5=xy|?>1_B6gbgt)E!|%&;X2~)? z;7OdskdC2qaymWiY$5soe&@YgU}lbm&M6~9!feZMuFHcDBb`7*m4wyR_qr%fM~rZ; zY>8=F+FM>1r4$iWAR}xN%gck8x(ziPwq6$%7BDR*E?6|1J+E&Y+7c(iWI|I;gl7)m zZdV<~@CT}$#9o1D35~uTx8k&*{W`Cdp$(4Pq2hHB`k4;2pxeAD&e@Dc+SU#D_6P=$ zmS}9TbY#W((lnY8h6QDf@3d)5oc5l()}uX~#la-%>Pgch_p~(s0#>*gq&hzl;^CM$ z6dIuc=5ctGi?Kz-fM7=01-&Ne53#v1h=eOzZjKu`4&k`|ktKcf#rSeTC1!n4k84eRs)hqK1Fl! zMi+n9@FkYyWr`Ea`YGyh5l;xs;bO;~88B2v&%cw#4;>YMnnS6APNPrXFov!YHJw7l z6oix!%3--qcREUIt(?SNGmwEKbtgFU%2WuKOHq>}+pu|7{Zu$T*gi6iGl5OOF(i zz$USPEas9*SFP~0ID+5A%3oEvah;dxnxIZYThO>90wPx8gjhLJ<%zHj==+9aTW#bJ z6rW#B36e#zI}JheKls{oh`p zUvdStI_s3m8|&T!;8+cit*lQMy)itFoMY8rNnZ_IjX6h*?2#MR%(!D%PAO<$oK!~+ z2WT7uNH7Y_WnI!{7(R%wi$3j1jqp!D)#PyU$h^86~>MRIa*(}wIa|pX9YwI>`I#*RomJyP8viN~}u>2cdQxz#R zXy=CXw0s#}XfYF{x|iEq&AYL!2H7`@iM~?)Ob6;jk~^}z0QGjsvY;$>k;Q{nZv7nC z-JXeXWcsbMqJ}T-ND^&Oe~QIed6vS#WZhlO>|@C_cZ_+6?eu%YQfqNOPAwgTyFwA| zm4IMM@!+T&1=d`jUsFkN(;@9QI9yt}lR73qCFqi{Ue|Qo!s~Z+h)A_Ia|IwHIMTNO zB;+-qgfF{7oJm!x5anl}bx5wwi=`085YpZ9u$ID*XzAfV5h7>Nz12rxP`OAoh~MA{ zBWXx_Ba+HmBywF3a2CtGw!7c!Du}Pi%ZUq#HNbIuIycJ?51LzT_RUl`x&-At^zrcD(z>8P8i;bk|-semr}12VU%J z3|9>R`loV`yUtw_fqp^Wbjb){Q+x5lOxcC?cX$4N5TNJa-pDo4;FgQ98j^;YE!}`s zUnlwwh$OQMCw0L~6&GuLl>zymoa70>*nb?m|z3Aph}`cu2xon2=sq{>;W$a{|LS z5rmy7z?>2M021jl66q~u^(e(u>jh-BfJ%rBxI%j$J|Jc)HzICW?!e`;GR>|+OLC8A znBfT#gmHb?Gd$rgHPDh9MbniOFv}9y;zpjKoJI>)qv=?aw3JS;Jol4iCiz=#1^!kv zq;Hi&a)adUBEl?k8Ps7+67AuKD2qpNTI{u-=>S%#)&a|gV~&a>{L>i2g1)>IlkBpf zrI$a`0r;*IcZ}uv5y#{V>_$X|FR5@TdZd@?b*sdzs~t_2$9$ZQ2rt;FW|5oWgubNs zsv@`25btg0QfTT80e0bhflY6VD_7E`SQ)J7?l^_h%+ zCE61|!h)O&$r6udC?3k&wO4-Iw_39^3Lfr1su(Q@?2+tmb-kP54?m`si1g8p=T+6M zUr^clB~`6oR?+%J)U2+OwW~eA(C)U;nOddk!FIEXwWO4FE9}V%UfBeo(8)8&FEN2s z`uK_-&yfQ{`p+f9%3%C;Y)Zv_!5mV$L{g-9N)Q#3Pbf~nUu6qjahVLaR;Ej%4VLgJeYA5E% z-bZzS0azE>PSBcHdWCPNt4>zisl^%$xCaSW8A`ajS`}2qmPK|jL7dqGuopjY^J>n=T4CT)MeSUdy z^7@76u>pnv*WGSmLZVUrA8-Xf`~K*-*oU5$=T{~;Cl)k;@K;l;yb#qL3mj%toP5Da)Gq2-IG7G z0$&e9Zpty%Avd6G#;7DA=;XQM-vtIXC(nyYuhkOP#(9r9cp!j5ur~Ze8@)|==j_R< zJ@{HR#(`F0KKX02`>N&_V7Uqm&6wUmcpg}!TsB5N#2lo+`jugIB*g#$dZjy%r!p1& z0gqr4)xvf_6pV;$nG0Y%CCMx(sEvu}!xV6DXiB!wOUlIswH9NF^nM?!+W}k$l&z9H zR)EEpJDDmw2)N;gaX5p82Q@TeOh;$h?F%$b?IZp64u1 zN9VSg6NrhbU%&W3!W=-+wNzW@TAX=7*n}6KBproN0YAV^(fy!4LLM5U5`duEl_Cfk z*;*f+q_vL-Vmbm>6qWRHN=3qHA%xavQ`taB@w<&GGKij8jYH(=_O)`wT6yWHEYIIG z$iv;CwmE$$q(!MK74NFE3*@PKHTh^K*bVlAeNQ3)GYEH~zodL8;M33~c<>`S!EqXB zD`s72G7F-V2dZws^WiZ5;B}4VuWbsf{nCgEt`{*Z=|1Z0c1lG^l=AKJ-gh@11fMtB z7=+PyHehjN7NMj8ZfUuc%ZrinzPuvj9~A#n{wPP#M|ct)?z?RHNL~(5Mp#VM4HI4r z^8^XPCUJ^QSRyb-D5=%2;f;IP0-A#Hfc3!JaPnC$U(f>g55!Qi%yN23WjMX)^J_?DXrf$Tz zk9rZA_MDG?LPzsEQn$3#y3%(p$+&yvHGj%t3aoyDIe!DU7^QTrx{Vtx&|bORMBJn|z+z6pQ!KQ|bjd`c+IJ`JN!8{<#> z({wX>hy7pQe@dql@<0AZ^tV+0lWod>A2l>C?=IjYrjllx=*8FL{+=jiBZO9eqt2e%wWkl0I zB?{_v(8hCfGkHCC+*XnQ8hhnSL zaO~fDJm%u4t`witPI79w%$VflK2;dGPh}pizU+qa2t3NvaRrT%D3DWAe5|nIN$Wyqy+T~R_#H_Z!(`g*V*pI@|r0@oXSyuj4 zx_^j4AoKuWiFg+q$Xc)XP%_7I4%~fph#YhVw@ORc19yhKg)%+RHchvmEtDL z(xu49ApcV5_{EqJKBh_3M|++kqW>U5$++c@mynCOP(fAf=QU{0NrGpVF#>nvGOx5F z-|-yGT%0a<;`8PiX!?H~Y1KlPV{S$RyQ^&I*N(a!f8s;Nx&yL~qhk7f1YVTj_(P zA>o3nq%zF@K1w4{X=&ryqKZ)+)CzizNU6Y9Z7?%tel|&fNeiGeM{2Hvqs|-c=G>W+bXVUH-@JYP;_~$E*~PO*8zyw^ zhzR8Sg++HM*=K7sqDq!dLQ@gogm|B*$zO2V0NQl7zA z6i46VMDlxV-Yg=7AOI&i5#YE4w{>MiIEm9-IDCrRBsw`YoP)9N2|P!k^KY61yLf|z zF$acm%EQH3h>u-N^J$3un_JD|<6;)iU!47Pa@;VH(EOdJ@ix$v=Kk?VGv$ec&pv->pI4z`1xIp{Y|9uN-AA-^51z>&x_ zISg(vi`Ib9M`pg}f7FbQDvJu^&Vl8TbAlgR_7Ew?woA#fUFYx$y6ly4TvV65dFVl*Kt2nFX# z*K~DtRYXAdC8rI>OOgvmoep}ht%!>&YR4QVz1wVb9dTny*uchR_T~NoK#FPja_9e@ z_`l0BNixECAhLO(AQ$BSJUHB|#{WGyJUo2h|L)`IqBFGuV6~}>8tN#+m`g2^x-_B4 zI*_=#LJ@Q%J{1WtXbkjctCOJwyTVuYV zVu44I6}8FZPRR;EIj&f@El(DK#ESa0qm7mnyGRBa79LYdDXneDC?{@<-jkl8qz1w8 zRqZRMb%kGTH`CDw0rkB+5sc{o)RixcqPtUQ#pUZl)7^RTOOBIH8Cxx27?;J;9IMOu zu?k~`I^XUDJNy222L;EDeP!^^d_c57BKSuvjEk~tXFvoF+H4Ul^C3&lEz~sYa*EM3 zp7g(no-GSO_mZWMKbYjWq@lK0RHoz{&zG?IcvY_E*xQb_g9b1%B@OYU{@No>rnEhjl`FW1==5@a`0{} z#))&?+H7k)b&=puTGLHO|4|c$G&az(PA-b**nvOye0Bn{AyT*Duzz~&}ZaHlFoDmShR7kyWtuMz&g^8R1> ze@g0KDOql}2!d>VywJw{`0bmOCIlW;UzMS)j>ru;e%3mGUNM5X6~EGWmVwuR^z|Th zHjxALvdib5*3KH1$yZq;ODu9*Vozn{b<(PJJY^yp=2e^sry8M834gWp{?ljmC zi!`lbW0kVE#cvhW=$r_f$sf7y`$_>;agV9tx%@p)4Jd=70J?6Zr*gh*?J74ka(xnc(c5N^J5zHetun$Irmc`N+LSsmpZHmf1hXbkHDP ze{J12{_-@+3FNi{A~PC&eG*j!%v_2}%`1LigsQ+>ok$hm9thNzCs1mpj4{6JgU~QD ztrzT?cR`J$Ji6RMk5pjB{xkH*-fC+#EBjp4vDe*_J%91pzEoujyZ)kxdDnoCWgx72 zB-|I6^8VpgE?g4aeq2X|YIo%t3-|9m=>P7d|J8~$MiVkG9pJ+JuiI7q@8QA0gZ}qk z9=)z{8lesHOCcW};Lu3j1k1^0>1(4Ah%TqZn1z8dv{Z`hS>)gMw9&F!MrSl3o@=nY zg6y+wR^jO|R@$Zl#<;MF#EPyzp)^h-k_xmBvSDKnBHIBXFdq*JfU5CGB@nN|LEWOn z8i7EJTt=obx#5nHh-^w*okQKGZmO7`(+khRK}ChiUEEs>U-z{a@xycLXI=Dv5T{X` zjv9}FMf87fXRl2E_qKO-b|2{feLTN0`v1LJMx|q5jilT;Q&zUs^W-u=BSQ$JrfKe+ z;d$02$2r}qT1Z(r=j4TsT_wv@z+6<+y(!*lbPj3PhXQ2&R%0lKfm^YoCY4`Mz^k)7 z+{&Ew;S{>-b0_+5;|k4B20owu?``kzKdtiryW0m3^#48{msH~{;{{cI4wHLM#M*fB z@@`+wpuyejYd#8UrX)>avQ*Jp9!6Q$5s@~uY-njlAi5-XVlyEXHSLu%m}Q4kx3eia zKQxkzrnKGeQPNPL1tXORzfhoZ%6^<>L7#CR&y49m6N5Xhq_9hU=q++|pG=4MIaHis)}9MIgjX+^yS;nfzPF>FD~ zBMmL2Rqj8ttG0Q`fg4eyZ_)ed4L z>vHaE*~<+X1S*xXuYF;ky?eCr^6mM><;m&h`~Js|vFYLI=lizjqpGQ9wV^UZqcJ;nYY*Gd z{&rjY&DDNjbkeVUK*~izU-qj{E&N7fWup( z(`kJV6zZ>QGHXSV{I1$_%&QjyRoD>BpL)f=l&XUNee)V?aX>Az0_L$BMP*l?RYpJ=A8pY7_*NbQJ!Ul zEf}>bUTvl^_*v_EX((N7FspNMjZJ0EOraiHEy`MJJJq7qrmCUYS{o`CEH_gRJy$$) zw_Ca5AAY}8hWlsPa4o~W-Hxj?cBd^@Y3ufTuF}vQHeIEqdAqL0f)?@RD!a3yEH0|w zmfNiCxVBi=&6{uDg}79`T&*qAwM1U?w}thc`(oSIC6Q5kS&lpF;@vL-w}q7*y%A2hJnlRU73D|ElVRFSdxDSdkZMv-1c6DMK=6dnXrb*gfiQ(m7 zxGwWv(2JFyHZ9IYbv_xaEE2H7(3|AJFpI>mh=kihu{AW1XIwCXC(oL~xm9j7y;^NR zpKs%u`LdH|-+c4UH?@+AX8FT7A!@2n7Z ze0NvtplyA(aF{jon|O#E!=<`^@&f;5+g2?lb*bAWdHIq<^$JeiEznE;dkJHpQbuKm z&O7hRg&~V_rQqqw^KuuZJz^8V+9EdlYY9(xwa&5`G>bAtQq8sYRLXN{i&gVkuCKpR z=J%ppB_aH(_GWI3P+O{!#`4x<(3$o6wF_HC(rYtsl_--|&y>nG%UkExcZ;qr(`=Dv z&h2p#3A&T0a2b)%Jm`vgrc~33;EDiCG4%DL@*y?}`j#QPcNOm~g9AuY^*m-}985?Q z=aW8qZ4x-$6~eDKRkwh-ST=$f+8l{9VL&f530Rvf*XRT-tnJp_tDw{sG5VO1Oe@GB# z^Vzjgfkpd&`)Sqx@9Ey&L;kOOc|LxuF6dJBLAhLf--p1;nY^$(LMqJiAN^eXvpjd& z|EZE`WN>G0=K&VQf7spLsqX)s-JOU1e;-ff{MR$8q#!Pf?(zbu0{VZrzgwmMhx&o{!6fGHp2r9J)JP7He zH{F5Yv+5W*wQYR0Go(43?3ai`(Qu%&pfieMd6wX$$au2&P-#3?ex?Ix1CU{7raApc zX8~BIt>O$T(9sq@c5aK#7_=sGLG+9z@n>3b!}=D-S_1|TOBgKK)U4?stjPxKT5C9p zA^2}`_}`KhT6VDsfy41@z56sfQ+l^@=#M3P&S-R@0s%X#yz+A8zGeF=Hs`ND*P4g$_fzcjVBV~& zhcQhlyND-*3p{aEhLsO%?XS!s=)N@VTvI>lM88mGdM>eZUazY!Mu35eQ#T@q^YT8{ zqROpz)&xoI)1BbawxmL2yCwcig;!XHRfoDogTpuxgq^7z`lTiz?9}`gOO>5AA&d4I ziDCw^B3A{pQvVjGVA=jr{k|pM|Ge^uyWRhJ<}VQgev$ll`=GA>IegIn+{^RPiT~@E zZn!cdN57Z?o^wFV89KgyL6P{x(Goo+&jgUN9(xy>%j8m5blr32SHLm&*S%=YqpNw zJF$I5&%Hdq8~VTAnDT2n2G(*4+&!%PUB^=de3wT=OUU=TiQn(8#C-{0jPnL;WZ&3aP!q#Ujp2i6ok*Kev zIvzq@q_QxFlo9Qk1b!?$@G7oKK6-2|^&guz#-YT%$QYHU4~gs><#1qsM)W4V!7MsD zJt->5wZH{Qwb#gTg5!z(YWzIpi;`2G4mG3Vu^ty=vd(x;fz!g)#z&Jl?IROfc7i|j z(c!_t-hsSk<>?XskrS<{>!MS_Ai1U(lfqRS2`WpfiDQ(C0<|1rj&4Yj1kixI7535N zGuiA>I(rQ2&sTApe=r?fjAO3%0K%M0e5-7#G?I-mf^r_GBXpEyM{GhFVpxnN2xFY0 z^B2%odtZ5$C1gTUffIcxb&I~vxq#vd*I+wprEX?r;7BrDDowW>>PWT|SjA-tmlbRd zx#EECvriL=2wd3Wgzy;`WMUS_ADi#E)A{c>%}j7RE3i=hcUY7EZ9jd;|8+0VGA>$e zt{}&Twl|-&cXh(L>#=J9s)o+#qfdVWLeQv3HBz%9Ay_=fh3j^E=X6!3p_UNhrtiK< zD^Ymd@_bNyy;XuQcMx~Z5%#s0?ZdOC=T7wB7|JZT11`G%$;)2l{&)D0|Mz~L3jH@T z+G6o(Nxzm3{UNScU)93D1^;pOJjcnoz~Ob-Q990kz>_$MiPE@NLcKIqFKX1m(cfkb z5!~i9H7jCglx!D@UoipU3F(Zx|KGhlj{J9?RJW3cP=BrFdakkuMtO8gE@A1!!FS*n)?&!f&M?@- zjqBqZKGtHbRrJ9pTm%48V!Lwp%%6YU^!uAEcKWMOg@ulPlQyrvWpMLF zoW<_oZaDcJ`5G>@_SWwPNbcZ(=X%o3L4L*mZkd6872mf>-SO>*I3bO0Z}P*P+-W9X z)2nUrWgWUEU)Gyza$hb_lePOh)w*BctF#ERIbKl9Et{4|kuDdEy+d?nZ5Oqhif!Ar zZQHhOJwe5`ZQHhOI~Ch`gG%N3^PTZ`&S{<9*zMie_qf+sb6#_*whJy}j#H&6C&}Vi z>TmBG7+zUBWMteEc#FFrz9czbvc~74#+t(*4jmPVYo&u6d4g3zy7#OopNRPH46XX- z0AKEu1G>)@-v($*ND*CdoCb{829%kGO1j2BL@3>IKZRTmJ}H*?5J`rdMgz2R}BN=r$Wom zPwoUM~^_2632o0I%Rsfz@bd?6z^13$@FWpp^pLoMzvSq=8llx=MS7d zu_95S4Lz(H4=!P=xiptqYMnR%lc{dS-8)V;45pNm|I81@hxk9HEU{OQnnWizzWp$r z_J}5EQheSwSMtZyCw@M#H;IiiP9$C0eq+Q52A<_^1Is}nO-%+*;UWvl@fYr1;I(O-WAyb0Eq^j~QG_t+*sqx&5G%oFI9qT5emS?5bT< z7z>pT@wf zW}i2uny<#&)EAN*$^BvhyV1--FD4mQl`+7;MoNU~;6zTield*VV`SstbGN|@66vB& z687^C_;tTC@GgjY5+Z%1{;o3n3y*e&P~pNl+z-fie|1G$N>c&zXpR+`6ibRPF}YRn z1ez$O$qc5jG9g?-Sq@U$@L$(TwP}cXLcSM35!%OUU~POW#(MG()R!9#+$*Pv$0XwE zaHMr+2oXvrlac0l%|Dn^kLq9)et>i92*Z=1 ziZD{miTcT7tmy_;>=adp&~Az&*l=#E!A7|D41sav%D?a{q(zyjL~JfFH6?ZnMZ|xvZ%lb|5gx0!s9rPt z7IJIs3#aT)@vBY!RaQxQ)&TTa-pXYvq4l9Cu(!4g;$18|oE)uJsv=au%ynhu*N%S| zs9*K&KVK4N;eOl34xGXm787%7ok5(rC2XZA{3Kmc5H%5QC<&gpl0x<<)M{n&a#xil zr4xz=ooh~pxqhOO^{KwGJ0Tp}Kpv2JW0#dkbP&s# zTB=aCQ2LqlBIl{k@|nGppEuRuXR(&r1}vW!6sJeQgm_J9w(UD>@pi&@0!Y@wEcC(e zS}kvZ)J6AexEm2_x$(M_61lLFzKw27LGJD&#R5jLQM`8kZC3CK$q@>HjMPV=ATS*j z$^6(h<+Tt&=2GH^+$1Qf>cS5U>~R2kSl#c;5-&B1*Ti?90hE{Af*d{9q$d6}3~rIK z=LKbs{RTA?!E!EI2*!cA-zjv($+L6M-wJN%N9qei-L`f|)4H&MRs1l&v?YidXzL6i zF8<;ii9`Q@!{e2h(`lsx>0P~1Xfnmkq4UU+&C$zJeDOQwPJOJ??32RS-dL-jXsBIk z(C2y&;iaUAO^fIeNk*>HwJj`hEW;Po_O!kb6Yw8_R63%ZYbR#zr{KO_%V${RqWnk` z&~8>4yMoe+YVNt|fJdC6Bp5?To95gHi^>%dS({5pMmA27XcQ)0+w0V`sp89`VPV1Sr)!4tRPYwG*BEq2&FADHSaeD>- zV;@xTT~+(-%{_w(co=1WEr(d`8ETj7*4zUksx`gf<_rM65Iz5^`4;L?dL#M@xI4Wp zKHrAbb_0k-VyW#zX+9CTg)R*w7!Eu|q$)HblY@43(Ldyd!dQYUu*6UdK){Cm62Upk z6{l8&(4bFjBTCYO+!5gR$8MKn)dQ=xz!;1F<%snkWcLt$oHT8N0P0WU614%#q8qLu z8fDOPxAk{i^`Ye)q52!Os53jU5)u;9n-?PR`z>#80O;?<2OoiL*4q&`zypm`fH=j) z*H_$dS3>|?n-4n2$bfGH6R-#>v#FJiG__-ORKwO|TpJ}9l~!SlD`I1f?lNu>ymGH* z38MArW8@W;nGLGXO_@q%j?!VoPTyt6f`@)B0B#e;cG{MA2 z4XosJIt$u(?Y@AT>OFFft)GG)qj9~+UyNHv9c7?{0!D5lH4`l`ab)fy5J_A`JnU+ZD+7W&< zdwQr3KDf47NJ3hLyA)_`QcdQXqGeH8)y>;r4XtgfQ=qGB*T6l7olNvRR4J}^9BwFE*B0u?)>6)C5DA?ot zd9mkvQIokXcqNT!=2da*aq2IUWi@y(zL-W=aFcht7h>>Fgs7n$e>vvhyY{x?NGwx)GO0G(#hf(xS%Qk#(1rywU)DAI~Nw0*&YJg+Ba z`|xg)+P)v`!--H*rA!1xAswrkkq2PiFuCrQnC>@8Tv`Z9Bzr}I zYg&RCYRI3?C#0EYJFoxWhS&lF+kH1=Q9^;hf@*U4tDV=u95>>`>2G)}vZiDe#%Lx_ zC)I_M&+otce-4VqQb?vLls}=@hUxOA#NtjSTC|*%F=A7)bW1UE!mFNzez&e3JzG&A z!wCLP4W&8eCNB_z^25c|dn7I*QT!wls&++XW)8b5c9ZL+Vh^7ADcIp)J#8w7&g{mT zTFZqKh#0NbbQ%vufpV7YJk+yYI~dI^3_YZgch07YI`@eOkph2DQlnHEa&6W)!lI}T zPu(-d{MBI$;gYT#cgmleJAK91qx5tA^KGikU0t!Kos)>;u_Iq`vOgK9Qpq3~kA$H8 zNfR%)8Hy{>F-0-iPCZ1n?lXj1!7j$CA$KW}lEUTBWk$e?eqLde>$ zsw8^cTrcYE$5$+IBylaFPM?;nYI5tufnXnJJU?qql5?wUh_;(~2M$A9XSym=uZlo9$gjsJa z8?nWoj8|()rn?8j5#8CMF@mm<7OY`sQ$uCx>N;;#d3>$}8>VVs_(mtzNDau~ylXVV zJ-Xy5(3?QXV@}JmnTNpKB5NqZ8r8IM?08bOl%X?gMi~Ns^)bzIGw7_HXwOR>W=X(L z$bzqcoYOp{Jt7;VC6#Ynq*?r3uGAHOgOZRhs4id?M6a>D4>|k#+Q_`9ob+VCU1ugD zXCcz+;xHn*EE*|UE7xb0j3dPXhvixMNbHLcpCIv+1uA6o_!Xq_78g z>|r|I%{@eo0d%%d(mA@n!%Iq5(DpDkFLm4fB5|z{Eo;9l_NQRvi!Jk#z#ecaaZcwB z)UKs2&y+zQ!KXixR@uFhT9D9t5Vt)r2=Hcgg!gKsZYK{Z9Ma3?4SsNU5wzjfXmQYB zWJ1Jac1*Svib+O~Nn5)Ut2G#iiCg*1!4ko2M<~9b{DzrZ%!4+`{)BxG+BCkv$V5vv z8L0QEuY9VeUYDdvrK4XX_Gd`0f_Q=PA|W7GSp*A)+Or;{ieb4pfNT8qlcDsMgsv9b zT;Xd^h(C!c^Kf4|b|(IHSCWGUdW@@8mP%Dw&J? ztiGbEPLht~#yDhZ)?cZT-w>}x)6Zfvp&TWglMu31%oK_?rI1g_`-wCrdS~M)GjtBD zDPO)2jnU*%a=IsQ7F4kE)Sh?QSBh|N{P+qkxU#{paC%Mxoylxo5`4Zm&#Gp`u_@v? zhz4zH#*PZu+sn7k{w8!ON?)A0r zDf42-y=j(qbX~3f?B2heTc>t`&~#G6Zf&rS;5jo@Xk&7w$fz#Lp6+o9noU^pnb`6$ z)2syraJYVteV_wx!yyx2wjmL;WMcnO`}@wOH( z;Kxs)R*K~fc0Fpp5V(|KHUJ;dvUoXbt$(<_o@ukABx~>?{qUk1=x1b)Vj}{WT)pF6UO;8aLsdEc4y_e5=^la_{7&%%eUi1m|Gzkf4p- zmj6G0Q~f9=o&9h7N;(}^$D*^JqL*dW^oY6#cp5D@>Bwt*>;uXXi8(aS|HZ;63SnQ65 z(G2|VzQN2aMFFY#jUHh%wRFVQ zQU#|8z-j-OPt)RSIC%WN)pJGve-4a&HyFyN+C9X9zgx(2`|Gdmf_fTC2i{=(k8ZZF zojMa*YO`m`0hsRJ*=NBN!N}^zuyj~kNG)AGUQtt;Z$-XC>7X_k=4tJ7dCnS9hueOB zskcJ9iEmnWU|rNICx3;FAb7D7=D6*NRt%iG(4E&|M|t?sMFe~ytfcYcLZ2zAou8=B zWkar?_~(iZ^uo~oQ2X!DX|9b(zSJA}Z*qTmfy= zg^#SrJURa~S!wnk&ae)pRmC5@zMcQJQfCG%1hMAR|BUBA-M05m$EIEFSlYt#=i>8Ay1%dnQ|;W%}Qa?X_mU!4X(Z20}K5X!ArW38srbH>l_KA(nm&EM4Hk_Fy!t5R=wt4bh`OWWcl zbwC*S2sOG_<}-G2TAMeVA!?sw=`r$t%7q~KOw;SQ87vy*Rl6v9+-5UxmNm$`FcR|Y zSbN|?6*Go7vWnta;&ru3%KOIHD`OGazCGkc*gl-@?vVtb;GjF7D-h{*tjY#NDNA;) zfwbW?_+M9e1Y%h40Yv-)l~=%g!M)#+Ku;#5S}D4m%o0ABGil$XY~@88&MW-B`3vvmR#&ta*sXkKu=>1%DfHz9rt1nO$XH3ePA@L-c`| zZF?oSDavw)ycT`=YbN?=0)BDr7`23+V;-Sm17y|APM>}dT5%fK&&NIQ+WY2TzwcO46IZ-vsH$Z!WlFmlrklvf0Q4hFi65JS7kEl@?w~b?|Hp z8mr80pSaKb#py3kZY)cL34JmqEA0MsWqJCTwHiU)KRl;Mn|;J7rxBdwiGhezsI_}O zHJzL0_!d6Rxt55TnB`p82%@GwHU@3;bl0I`A}_knO5QJ&P5E`2xcS$pLenH*zh>i< zd18|{E|vL7wh`}oO(<0U6iTW4yMCN7DI)Q^68>kx4O<^@H}zUpB>0 z^(4=`mOpo$v%rVZ?*-dDd%^6#&E7zY$T98yL0Y(Bxqs80I%X__VT-5CzG0kC2qa0b zU8O$oVM56;U@pJEWqC+x4sdQu-=B_-T8CJRQ5-`A`jRjWc!cKXlpFj18u`i>N<0R% zdsTlJAr%lh3hn;bb3b_Bgi;_%%QQ`v?>Gxr!qE5M>ql#{T1u}|DKx1 zD17{f0cT3HN;|i4~b5Id@8dX4j;Z1&)FJEWUd3nt!hLFvv5^g zae06sN?M8Vdei-Z7-`~&m;1X3WTQ<{Ed-;MGjxjjuHhjL}a;8)?TQ{UfUhcvNnH0n1gudX2a(T2)ta*oqnJw<`vra~OPEoL z3+u&AMj9T2i?rKcR5!eqi)i`>-HhMb(${A&wvv7K!c5-@5X%NJEC7gkSx5*)XMq*V z5`(yHH{v=q?P&{iS!|tm7!;V;Vb-qA|F_#^$VEyK;JZ658OomX3-LpjC(_k8WUN-a z?29yiSjz7PwQ%v{aY_e<9A9mfhbmL#iI-{{F1K_kR;D$5;!54)Dk9%W`4xSkE37DpRhD}B5qO(kDB}rqn_dSyBCA9oZ z#qKoV%H@nyj{M>|XyJ-5dh}dC5zf^?q%`Qor$LOEtN6ytPD6!44Vm?)MFOHaG+)kT zZL-8P4HYQYGCgq%RmTm>5sZ@fB|XkPcc6AQhNe#)B%}iZt_0Zh_DeKhPh4=j#)qvD z!PN8SwQW$eshtW^jq4pn-O*DZURohvACCJ^3vhrO@n|K7$2(D|u4?*%7J?vuK(H0U zr;!A)UR?mzHt_ED{$A@g;Ah82_%iVJxT^@f&Nrcj#o0YO2juNXd^pWT`6nSZ-B1Dl za(z2I907GD8|(^Vm>i#UL%azv8xXAD5wLqXh1(qMh9Fix^jP!1J~y?u-Ix6Vcsnw7 ztrho_F%HxlrkdK3u9(U>Hl!)UojW$S4}G`hWT}p`-G_7@v+s-h5*2oMkN&R=0 zP3xWInjK7)>w3f8#!IKiUJ|VgB??(x8<%ToJlC>ej!s-nZl(_D^f&II1L3A;y~gT` ziIo2CF17Mu>a~SxD<*mx`_eoagq}TmpN2hE?K~=l-L}}KcWFmT0W`w6X#>~2eRF0}OTg`??`TD>! z&T@8m+7T^$!%jI|0Vdp$7RdGXMeu^Z19Ho6OSaq_&%0(*gD8xmIC&p zPQwZ{X-Uj5-Wh+$(wWzkOc|wV)J{#$$;EM@%h&v3d4Q`S z#4B!}ws?L)-n==5Rs?=iEXRZd;si^Yf$&cd5v8>WMZEJ3x@SKEieWF@rHmU@&bXh2 zfq%Dig*m@CuWr$uyV#=JFTjMy!!_gQ0G)GM^rYCT&BCa)-DjB?xshX$)QQ-v#-!z* zB}_5GQw#&B7@de}E+(hJ+2fX58YxOLS9u9Yo~XQgDHjnhL-KMzhm1|pav(fS%jvYv zI%sBhnAu<1qD#VjU+;BcAX|PG+o9%5z+dMcMIaKzX$9G+}i=TurSrvvN4w`DE1tpJh zvz!{$l+D z|0e}CbK1UwQWFLP5|Asbxegz(;VtfccsbjrPqB3znIMx)+%=G>ju`lGnF)^?k7K%MH9gy5fmDg0Y?Fn$_S8c>c_j3IFz7vxm7^J6dnEMU zk2(_P@zEmK)Hi-D#`@8xtZ%H64`1O=uQhVE!CeT*v=e#lnYWQE*73~cO(7wD5_n`i zWSx1W%QC`yOHK7h!~)|Nd&2Duj+s|)yE3+p`ei`7b(RVWkJ-@OLeUiCc3Y$s9VrqW zjghCwasH6KlcVCtpVp#(jCU!dJFpo;<-e8mR*~jb&sgE2YD_&TIH!(+=7=9Q-z8GQ z^r_G+%g|0S-1`cCa?Pp8;vT1gNN<+jq{BJD>*!FVa1r~U5^r(OK_rlw3PVlgBA~_w z4=de8M>2OeJ50D{#j(Ul%aXQCa-NTX$)RmSTA!2|1d9rqqqnR48lAse`B@1gdfF=M zpMw>@EMcZY^x=Nhm{aj^&G+C7lO*xe%1|E$N@$H&T-w5{a#9`r)Sik!&>EmEvX4XK zU89$ZrC*R(h6aI)SuN)s+NCWeJ12S?Wf?DN!G+U#!Ow*qJD2fW@oYzeprV$T~XzBCwd~)moXg+!OYYLvZ7mYy6yl;z_*-hxlAab7NFv+5% zQ@Ne-^Lh~l?xm4w*D7fl-X=?=WM!%~6>gWvs6}C|3ozwpV*zwF55A+JTS>m9sI|gy zIDa!+D#fd(bXZQu@Bzvo9BJs_3WnhH-o5G?1R(%rRqD*^aa9Y2QdAmb0>BVsnRzCo z{77@W^?kv@05&hiPM{p_*apH3RkPHd7F( zdM@huNxX5@s8xfwDa0Q0tkFt*W5t%}^2=axnvyJZ>{)Uz3IYm>^hd=k&BK9|JQ{Y= zB6--~L#)!XBfIhhxP~ecY~pvVw1v@(FF{h250u!V%?{kPUg(2aq!W*ZX0X1YX)2v~ zvhq0kG>6hWBKK@L_s+-PVifjDQz5VVmH?7 zyL9l6JvqaUZ#52M4U+D3ilKAQFN^mNYfE+HW3&%h9#!Mu*<&iQF$#ohQMFM~G^M%H z&q-6s&C1NPuEgzRJ8cP3kyDLcO-g1@ar^0(7eCxm`OA4JqooKVifA?6b1@Gi&ce<4b+ea{zS6 zjd&U@RVU(%%JXI>Z>=bT>flhNB*7iZ-%{JvDdgh{#HYcFjphRC(3$#4WV;)FwqR4K zjl^A@t3F5ANuWGXSJ3Ru4ARXLykwD+I<1PX95LP{VQY=U@!mi@$q8z8zt1i~2Q-~h zuZSqdr5Gk0u1RnH2I9?$tSd2*cmz}K-6QxU|ILSH_MAa6Rz*c&XkH|A;`iH7D5e5?;JefT)bvCy*r996)@Gw-| z8TSUuTH|@e(YDDNne0fNm2_0;Z25AT(kk-QVhlt$`2pD(aYD*<`qD<~ll~d+S_%Bu z9y45!ARX33_9)}FrBZ)|@dYHr0g&1Pc8F>S-7PFVU6GV^j;H0B>-WgunB zX#_IlV#tGQEZD;2d>F8GG0?9urY8gcy4KE1AahNOuaiJ6%Bmwo>n4kCU2%({l&6hK zKiO`~7rp;yK?=xMB%xs6fsm}S{AKlT`Y|VMnu7SiH8JQOBVNPCoI<{#LU$>3$FZTU zB-N$aA>!AQb*9PcluaFzVt%Df$1mw7HLFqC!+u9Z=q_6Z!`3yaomlPK2rW%(UivY$CoWcpTS(lQtLmPSkzgX77((6)qEXBC5-fe;DNRjYf)*&zh0J7pWo{* zaIsGil3BO_R|>GeeF&59x5DUo(9r@UfsTBUuE{Kj0X+LecoKh_0#5oX7+ix@yl|yp z712D2Wc#GX_QKJ$GFPtPw_t`y#Spkwdxndc+jw@KEWA0No!gH7RHwW}eYva<&`4(4 zMQ>LxZRS5R)J(OdJ%|V6kbL43EO~i-EFC>E>}E@2xU5<>7s~%y3CSsz^jxTlv&^B9 zU#s6?bEBf%Y8CvYEx?>=P=o#*)k z!B0-EtD(1)fYrp^aCRR)4*x^w@=Np_-*h(6p-1e7QSTwX*mTfs)#>QWfW}Ozs};d`mJ%O+}3Q(f;r@LW$AS6#a+IOWm3Ab){Dkfq~sTk?3_2e zqcXo}#=@T>;azkM&+DpI`lNBU${U&SH0fpY{L03dZ0YmFOp5>mgPtz^Mz4>K?D{X1 zcFtfuw(p{}=Cy(p7l>{!&lcyEK@h)gEU)L)WPFY{ILNfZ$BCvw8lf7z4~>pIU0=gF zy|UwoTHbb(`-_T0Elzc4MC!hK_~#|gDytBy5L{|5Jvs13Uz!~mKsrO|Ewql~$jR=7 zOGw$SOu1(4Zrh}b^K=ea+W5ag(_KSG6n*znxyrxR^PBpx)hG+%I?4}igSV_Ao)41| z8n+L#mJiU~IPEEbH+{_0#6~lr*;b$9qT|Lw05$@O@CuPET zVn3J$VKp>?=oXOde=`tTnTG)&%TG}`9X_C1`2r1 zHNudYE}#$sA2^kUqW0);TMjTjKoo$=M5zBOVNs08>?L(gD^6wtGw1f~fs8m3gtZfx zgYv2HCtOUw=Z~`6bwl4V@h^>v;Pc$WK<&7Vh4nVT&2yGk&%&!k&*L-h`$R3_ejOovNUB-V4pWK z?(go?`3QeKYxi?Z9xUHSpLLhAu;XjI?X#+H>bfR9{*x|~+;(3JB|cZh=GD-lml}P` z(dogv_v!n)A=sP0@mC%SSeg@<^N@}VeW&=phVUAC@D)cM_&MBC{T16e zUw3SohPLyxrh~<9=Xlss2+mz&s9=ru5?JvO_j~TRgo~9z)blXQDf03kncfcV#XbA4 zfRNwoLx(OO-AAP`s$FGgj-_1S)=;1mgN>9nwMKurj}P9pZmd)Ez{+?Kw~hz=Hu7tv>VUK0t(Oc=tB+ z23xdY0@}DQ#aVAW|^WMwli$#H?VNWC%9KB--=rX>mM&m!#3){gG1G!7NhLn zn6_mf-k`(hkdgDn<7=8KR*b|XYR`rRYK{~lr2}^PMjsm)KHN2DI4VdD)^h#DNQNlB zwlf=P?9G+=85lA*LbF60R)?`|GO!j+4G*~WzBCK55R`}DDAgO1H6x1PDmo(>nPysr zB3;-!G76N}^>IOYfQv*7b}58iV?EBGQNiSfLMyFjjUm(kbao6!=xd}z##&aQ9+DzB znz7w!Pk3Y`3bVIf7v1D4+j%O#xz7oP6~{zwe=JxU z3_s`rRBBi-$E4+T-U+K#*6cqyiU6;?>SPa4zCdOR4#WlA_;my`RVp|cjcYQsK7|A? z@Cq}4-7gtOO{K2cPS^P+5os;(6isZOlhh^c$|(>WyCA|~^5ofWqglGbafKRs+BD>I z@y{yh;Cbl*ejTjaicnV$RkC=u69$yJm*e=(1(Yro;xnI{O6)rF7Oi?gs^84e?g-(I z?0LGCJKGe(J4x}RtrCd%PoC9S!fieS;jSM`Ebcojffyn-`?gJbE=-+XgE3(Yov;y6 z2-s{*+UgpbP7x-ZL3WD`u-jj3xq6xxP7E+MM7S$~VRxZ#mN z*BlSN>)~b+-+muk5Fw*H`jLDDdiVL907y9)F`;Z&X{?)|Dbuv&2PX=JLEdH;*B!pVZ6;2gDYmE_#ryCq&6CrlAh3pyjB3IhbT#rOUEpLY_k!Mg8{x|N7uyb5 zr(#oCwKaJ&!0;hb#c3K$`>4p@7&kF?4;eNo+Iw zU{gI2447urZvG`G-dC=G5J(u$V2>de8_U^~H;5g{r7{>&tfTHDM=LYmPJ|mZW?t1r z3+tZ^vs$gif#{OJ+t2G-3!b!}%|#qep$+hiPd5w~m{B8y1p6bMACQ?W(@ac75z7HD zmsOvLv=V0NCE+Sa#}}T`(umlh7}B{-wCE=enGf~3gWG0Mp)UH`mv#onfh9&6Zg$No zTG=Cn$B*Y7YCqbm-h%jFn2OZXNgZp>EJT{}#zrRz)K0L)RU{iChNk7(EKpcylFO5Q4&hTKtlYctJ3 zI8Ic$JV(q+tD(|?Nj_SF#w#>2(knHI8dBBA?qf39Iz!ckuT>ti_}D9?9nXbj19A+-^ItB;`5|d9ZE|u~ z;LcCQ(|9#ObE_B!SI04CzQKv-ni}jN%AzqW1F+mS9ATra8YLW1m4`QCvnN5KFsUTZ zqL|;Ok``kwh5->?mb9xfS)o+#+r}0Up*Em)jY(!21we~zB^z;~UyrW?e^Sh7W$gmd z20e?3XD9Bp&ZL+Hp;qJFlD(OOBvNH+pT+4m@y*=Zs~;(RC2e=X9vJAYGZgB3n)jUs zy#9n|pcZw;N*ay*wlb;KOi(ieZ8UidTb*2vQ!dlFgEKE)qMGY42 zGrH#t+Jiz@kmr2>yv&-V3a>OmhCu##igA0CEePZ6g0+6{Q@vER*fHft1)hCidK{&5 z)Ut#owj$wLceuVeHf~n)-uN5W7$SP2nYY#Ql-L=2nE!wH6h;eHI7&7v9Uk=Jje`&H6+_?Ad@aR8l!*oLqddm=Tfc@b@ zQrb)7S zylUW4{a0@xWKQ;QMo5FbkX!c$>1NViSFug%Wzx-Ey;3Hg{G~hDb#I^SQ(F(c_BKDr_{XWDtw)l?PwkE(AKxrb>2e?&-40T1@DbG#&XNIB3RSb3A$oUyG53BigZatIJFM4b8(xj!(+I-JW z3KRCupOt-ZkMOJNnQfD9+9XgPo!%Al@fMJzPf5e)HNuo}&~bAA>5%bP zR^Li`S7{W5pw*RZR-kc3y{QQ?bk=MVtsBOS;mYsp;#fn~IwLB8c&y5$K zF2&S|n270DKO_-jD%r}(QG7-2S$$VG;Dw_Q;*vaJbK9jgzeFBJ49!}dzDnD^tuOm5 z2h^A?LT(F>R2@t?pRM2o?qvVObniK8T~D;7Zjd1wNuMQMLCb0mCCBE4z*fA8rghZ zOe%znIjk7KF`Run)|}VRL_!viuV(CnLI)jptRs9v-};k2U9MQ$v?S;2dEX+CJ`axP zvL9$E>~*Ax(7Tto?8R&M@i5Hi&D{Kg3BF`<`R`~bzFMb4?=J$VMsT&X5&tF+&%Zwh z-;aMLR=9fFzWFNqRK5lY3_YuVBUhy!5+7Ks%VmU0SCEQbzRHae1 zZ}?Hk!*vj3{4N@{QTa3=;B4GxvEy2kLHm+zBwiD$&%T!BlZF!1YQOh%?Qy=u4C<7A zIz-*x#qRa6#Pu>}?gCdW@pcK3$Krx)DpJC>R{gq^DEM*pMxvkimHs|oV2&`ae*GEL zOy_>A2G_qauDfe}`HF+=HLp#kdhrspJue+SPJ(R<_`bj#Rs8fGx^hl}jepZv*g6PI z&{GtbT2P3xTkS^E5!)wYu>ZnKVw9|o{#|d0SFJYJu`SRL4Lwe*(jj)+;yu-p>`!md z8`yQmuu!UN#1}i4U_3)IGGx3sxq!0e>EWXrAHweA%+sVh;Rj{P&y{^CaWq7_+1!>h%jz`fm=LjQ%r zmhnqsKI13zR-65JRLHC~(o)9r&fGT$jl<|-S}jo@mdS%`>!Z$)JX!K&$cZUI>A?T} zzt@~&B&URaq3=~k=!|c|um3uZ18VKVzfgVnZC@{2+XwRgJse-o9AhDQmdjw$EkkDh zB@d*|rU?Sy9CZ?tsgFw3r%bdRiCxMB$HBSdJ;wI#z$s^$dM)X6+p`s9z-Pdl!KpPb zCLd~OlNb>ZN(%VEYC}9&7TinX(4Q-Jy zX96H*5!z32OhP;t@+9AE0^aW~UK%Fu^|Rxi;xlxe60fiSgD=_2oj7?so}k-dg4k1U z6Pv`dh8mn}aqP@#VBcv7@c}%azLzt^c4%U`{BFg(zQy9(*Nyp5*uOw;y82tMc4LjZ zfQNxbvEhKR*FL7pq|>zT_+58XDMnb$QaBgXBJLr#sczW7xR_0E=XPO8xJU7ye0h^1 zF`oJoq$?dmaT7Jyb&J@Twb@I!f!8GwUd-^t#uf77-C#+jNEh3{k@LXHVtbuF!C_Lb zY_B7yHFx`ZTw~pd;WMoxVy7oe_e{=+a>2yEj$fj!dTM1r5)Tg(wT;7!1z_O9?bpC0 zo|Y7_J`O10{q^p`_WkzwFogxY_xE!=19~hl&x%5Be^u*MyrF#WbdUDWM`#aF0nd@X zZ#&0;UBQO;ehk0g5;rhMy-*&vgTjS`2koHLy$BdOEqQ>_Ln#GBcz6+t-AGQTd^W-V zr8+hXhCe9|!`4+D{=75Xy>c3d(}ll;#5w!HKAapR20fk*Rx0K`1C;Xpl7M&T&pIbU z1m7Fc7d?FG{!mFicHW#isFe{m`S5ZFyta-Xjb6y?9lg&!v-pqenVp*NK0N|`UihFJ zj3fsHs_RR4{UZmuZI@ai-4Yi&m`Qedsuu+u&UJTj192NTOU;zFa}KTkv$~F~{(Fq3 zbt&cT2~5`g9U|^Syr;)XW36$8b8vMVG*{CE-NUQ5Gh#Qn`Z;aTd})6gO0oo*U=5!l zE6ASiD8gY)>I=~X4x6p9Xy6-D0sn`+bL!5di`Hmt+qRvK zZQHi(q+|PyZ95&?Mh6|+9kb)SC*Q?6zvA4~ZH*d*y=$%c%r#sw`%bKcoKN)MB^|nT zzY{-kxL?_k)Ny#^9Tb-JvK&AjqU>)v-Q(sdyM9+#sR$2m20YWvfTTAD6sG%}uv=#}Dg5 z+yG#Zz#;&$ddv?6^A3)F5AW6K#}E6dz{o`8zN5gd3?(H()l zgVoQk@4XH2(!6M*T6XD3Ah((9JtyUW=^O+CS4oC8e%~KN9$p`Z+RBK9bF@FyR@Bu8 zO;s9I+V+OGQBB&eR{e87v_3vNR|8`|iRZL=mFCzD8?VN++kpiH91np=1xWcn*MZM> zf0NO5(`yH&JUhCq)ic#);52mH@C<&tQHd&PTw`jRE1|HO*~lJ!Np>p&@@|qt%39=^ za3N&)=54V21OQEL3{~AJVrZ0DX7-Q39mJ-3j`_Xs@xqh17$;^$*nPb3&YJfjJ_7e> zQ{_qh{ru!EsxZv_N>hR#v3q{|N2|riz;ozd4}Bj}ftVROyPh0H$w%HO--r#|sK&Jb z9QEUnA4a;{D>pO1?Pj6hPvG85m!b#l=&8nRKip>!I9a?1pOU#C$i@63)O)Kc&TsiS z-`C0gV&4Q*N=!5i2qc(0bQss}1wJE3Lch;`3jKI-{o0+bSf#tZ@{#JIsubL<9I*X* zyYn+Zcs(WTC}uf-o=Ix7a4L=UB;P5=21?}V9|ADNK86m;vK2A-VLu(;Mr}7|F?v&7ycLe{e3D$ z$ZvILhv$~j$#}>`1q8Bi3J3o%zUVkf_D;ICoeq}KW(CO0-w!`iv+0U(=?Zj4ZYjyR z2UP&6>eyfzH;O*l_w)?Ob8M52qZ9BGLf#T=D_IQ?!o^+Y?G!Dz>K19DJ7HB4L8ga2 zbP{Onh`jnU>n8g?);E*!I=G`|OI~D?_L8xg77Q!mZQXc8BnoHvg~_tr5a9tcJoe=9 zBhEmZA287SjgX@foF-?9T#dE38i&NOdFrA@JD4DFvBxTt#b`*{Y%5;bWB7$_-6%pc z3b=IA3JN3{u$1aj6)0)V;tH%*klHNjl-oSL12O6HL9KqZDH4gR@>>3ewS zClt||mD4<^4?XWQ?&?o_-C(pF#Yes1%Ky3B>kx6uJJGe*k>x$lNvcuqO-lq)YU#g@ zF{w0*pZW9>y)I~uoHBheNA7>601thvC>dp6-2kmWcgSy)yyaVD#T1vHz~I#de#y^} zFC=|7N?LU_>rJ!7@6-y0h4B{o|f)+ zv_M`)p;1%*?+g`#6tYDLp2~8pCynDpz;x(xLO>N(voJg^U`9us?9Wkc=DleGKTS}! zEL;c`GZ+@XVRd!FAU_Q)#*BwqRWp@pG)7cHk8y@7n>=>8iH^EGfiLih82C3pj7Y$D zT8_*2apFCqr=B*k`d@8oB6c~xJacHyWel8X_?`+_YP6L^TT)DA{L^3UEDCZpU)lxQ zdF@FQRY~WNwNL9m{$dTVPpJ&v7LDzD8YZddOmu&AS%b(|@iDB95Y4J&&g;j_BMqA6 zP1TWDDs&{4+(n907v$j(ECJns1OVE!cZEFt?e63p241o<6=c6LF2#w7V>uFHLuD~8 znU^l6vIO?q06DrhIs)lB7r_c2+@5^$p?3poxWLd9%@!hMIYNEv3&jKveKbwY+0Ysy zNV5Qdd(c%ji!j)fIC3t@{4#n{&>T6}1Ra@_mNt$tJG<2_h6ESBoT0mabk1Y-($5Sk z5OWXiDo)Otiwxxw47ZuyRgzHIqiVA;N+g=F4AC(3rTEissIq`2)n<8*VopkZPGne^ zsRm!xri~vUdl<$ylTI6Y`vxiK0wA4K4Vh6f5KOr#cCH`Ea}TM>b~o7&_hQSy&^K;t z7A~~$OnvkpZ3=Z-jK36dD1He4O&>kdjaDL=oD?LWZoh_{Hn-A55YHksQ<6|PNaOz- zQ_OA%OG+~pry+CRpuyDSBx%G)9$-d^3%l?&xO8fkt6Fo?2{rWrl8)nXyN>9qFxDre zS@|WRL8u#ju4Vf@#@55Db_M4xImhVw58rR8x0F<+H#fR<2_R-7U57RW>Gcvn?5Rrw zk@{L$f33O}0gpa&1DcjWK=_bho4f*%yrvhal&V0?Y!4>PT731ox+@tr&1qAk-v$gZ zT?qIh%Ag(%klf*q4B8=8lBb0FB*-%Hudg_;(^4Xp_ z_@J$ltR89#m?;^74~7kjp}4DR1dzr(c_; zbk#bIux@&nzqs2Ete55U!t}=6_|j|A>a5Q93dL^65K-88i!Lp8O%^Na1Kp!P6bd7? zHgd2O3RFuJi|S~y3viK;uDcfeD`W|8+pLH|0}Qy*+2V~G){A{q zho{hy^EnU7ohZlp^78fyV+)LNjl&fuVJs%;j+$6*2?1}V^$UXzAnhwWL{J5S;RkGs z$A)6kg&TbZg`%UMO~(>u@U27S_c*&wGjp@+bXMMD6rmJLsrl1o5@>^enzM!|d$ADe z;9{QOEX)T2cr;JRqH@g4E|S2lDXXC*7-fp=J#vd+mSjySd|K^JmHpq@lKGV?lF+iR zz)IAA1^xPY{4y*1ZT@lMnHYG68?WYQMk`-ctFxYRXkXn>nfkHbHVop!F~f3x)Utw2 zt+d>-2~xUN#coRJY{U&6rr(Oiym4OQAjz;YOwX6}JGaqMMLWLU;e6CFq))bm`anHk zButEfs)b4a7f!DscW`~l#HIFMwW&F(YwK+^QW@_tv(XWFtfB?GA{p&z?>BpB0b@mM z)fVSV_*D6tW-vfoAAc=(M1*^&$;=%7C#vwkn-CVD zpR$A%=)rtszTSMH{?SaxE+Ut7L{eX>5sdh!{`mz}jP;neeYyM6?hRj=vRBX*fgGeW{ zP{QKF%(=Op%*}0z?4~nQ#H<<)_sXBT$N<1uT?hJtJR3xmuZBP5+{g$IC!r+L3KLts zMNWDimFn4%&rqp)0aNNIgQ@6m)1u~65j44~NUJ05vYw^w5YHbv9Yc232$#j!vFHluVd;_Qw8QV8pk(PDI3{$`t zsk4q!A#Jcj>GH@M9uFo9KvR0Q*VDlkVk&|V9l)+GXU3(#M*C}X%s^FP&^oTqbq!*p zy{K6nPh?YN(nC46=Mj%uFCscMDWLuLwi5_#PYD7*kYpxTG_Zb=LDI+D{ovhq z8Fo(W04Z`)WcN+5-y}5kS`YC`Wn^!Av=ubjdxLnd4Vwt|ksKwGK{ytPBO_wDwqvZ( z>WxI?(!j4_3dLn8NMK?vuP~d67m*aT=#@)`$3v7>u^lsW4n08a@<&33JSeYh)>A6` zCW7+CER@*l#Q{_g;^79%;=GX~4DT1`8?(nDz0>`F+>a9*TT!>N?~v%Ma1=zkM7 z=7i@!LG3N|cH~r~A60f5u~b6D3nf4*yy|REG<01pZ#>xU&=Crm|7k+>lM;gLZ!Qvb z7XBMxm6_NkBk2TCsl9}a@|nij8Z}6U!+uhgHH4Oz%D>(x(oNKfJ*`_&I zF=Q&X4u_R3C4`T&eq*=`^^6VEJP!|@IIO9cA7=^g3_1;5`LQBCxcTytSvu~#R@yk1 zqGb2_zJfCmPDXHWpK+Llvo>2tII$K9%0#_dcIcp%0=ygQ^)>CwUBIEs^aF}0g|II=-1@WL{I#>>dU9wXfq zq4umlY$DGCmKeXl<2f*&i0dKnMra4<3Vb&dOtKZX=zf5Sww6)*Szj`Swo?G=M~F%c zh<4`+cFvK@TRYt6W}aZ#5<-9;4Dwfq*dJ8>dTo5rV3LHmKj-PE1EdPc{e<~Bos6V> z?s!@X9OxtTMPk9%e~z6JCp5lJSkZ`*974-JJ-HwuhyZ791LY=v%o#$A?C=~eI$1ID z8y5kqR)n1A83b*@Yd{u&bRH*qnv$54^d>6WU+QiVzKIOQv)XY5@=4vEaJ2aXkqZ!r zxgYwki3ML>fj5SN0_cPBNsJH?LQ9bi3}P=!A_sL z%rDaJUT~&IaN?;i|L*VaAMWozoDP4U&Obw|aFnGkhJptCqu}-DwV00x;M2UBW$|YB zKk%tLlSdfW;BS-H__v~e#Oqh8O775QE1Kj+i;!2%kWPN^K3lsT@725WX-8~0e-S{1 zICAklUKX_DMhiKZ^hN_a7Vu7oOL$>}@zBRn(YPVb>s_s@6!#7Q< zGkDChMI07NDR2(6LJ}1#p{$;DFL4JZ8nM-9&*8~&7-G>HI*&3J)dR#GYnJ09>q}+% z7umPERMul0L&x5X)J%}PZw#F5dlMmS=ps}Zaq4UPm-1P3I7#g-?M>eD+U)8b4r8h?+zhU=ZdL*(G5F`ar_Daf*1NV9Qc<-QRifSJZ4z8RZF28 z>Q59DN6V8FXAE{sq_`bdzgOQBQ=TRWpdj9hr{0zc!mm zn-+i3v>>;JjQazBI|zvEP|vgHmd5_f%hYm+c6w=3zoo}6`yAD^5y%t1eI49)JG*;I zRnLN(kjCg} zJ0zv_Au{v`0N>Eu`1VGJu=K-+N1)=ev>B@DYTAccwgx!Q zv>>}G4*YIbEtFBt{s3V+W`C8cDa1Bz8=uFo+%Un_xmw`TS8Z-e+wm4O* zX$KwPhkXg>c#Cz3#w)b%a+?6XJNY2C9+pc1g=`fg-pp;; zikNd*DnjqeqzUm!$uSS-AJZa-20K&C3&~GX=p?0}jp0R<$!kqS{uTYzO~zA@MmRjF zqaCSNA)M+=BuTrX*P!@P?s;!Oq_mnx9B=BfH{mUo^{B1a{jE7!a z`{Y7@(C(u@5&1di)u;gBAGFcOcR=>=_i~92PgF3 zi^X#AoY>9R)zgHM4Auj^$Q%o2U9LPZxuO6MzQlZ;BcW~qYcRplT5y7`dm z&oZKJ&{Dd1rKvJ5BAca(C`*QV)vJWPkWK;L#5Xi~?7`_687O5jqBJ{hdC6*D<5%J; zuVZkoyyi**R3M{}8py=>1|2Ke*Bvva!Y6NAdXR~o6R%y|&RB+jzTE2;&=&`Z7#h4^ z#b>Yz2kVQIwpCB*AG--9;)NzY{sXx2-ytM=e2<1M8KEOZ8wo`A?NRWKnOq)%YWZHRa>q(8`R*LWM*u1vgWQBU@V4xj9?rdt>bHSN^_~J7;EEA z5{CP~9d&8(J$U_i;%jweB}>ob4wmL)`xrC5T9YNYahIV_fe)fI?mLqPABv688A1qR zD@&;x*|EPReKMWSEy8b$T;IOpMbKq7#gDvupWZxwTB5({smRqbioT^sjhVK7)VTD* zDzC=G-FD!xNdpUovRJ*8&dbEhh5r1G{ms-VukgRc`G2IrAe>)`w!_X)1v$$uW)*mG zQ~5zu8*<_x{C46FQ_FZqNF5XqzPJexE#U3`caQ~Ro$sVV?Wi-1Ck7+)UKs32?{xze zTUR~21KoChY|Jg9S;KV`@A8Lcbv?k}9qZH{J{q(!7Eh`V`H`{e1Xmb#n2DgC_J^;S zz9U4g8s0Ji(S)+n?f`CcoYasHx^6CiC4{fDK~}|P3;r5ha@pad!I$ZhfW4o!pJ)2> z;=&5#&rSy)n?(9uSIhNDwhR+5!V<*jPE74WjA-ULj`xZFP09sS00xbnZv}$i&~BVEHIy;*ah4E?Ngrqk@<7XJnZkTogr1gFrX)7S{KW z-~1UhOLuyHYsWRTlYx`I^#+WL@m@+BS3WtIHN9RroHZWHy|&&&_AA3!^jcywvnz9i zUA?|tKd^I7kBdukkZQw5oH!pQ700{thB*_{3hhARG9FdfQGPz$E*oTh3uHP<;6!CT zAr>+W=)2wOanc?41AmLrEe-^GeRof&r-z9qkWfCF9aq$q>q8C7!9rs-MXtH-H?1mm zeO~Hw;;z)^t%mcvPrm&zIj0X7VZRnu`u_+M4$64(=J?`ucsvZQe7SZWtrU8c8(ox- z=q_n2ui2kv#QINL*qo|wn*N}Un@nPAJ}UEm{3g!1P`J>lQH`Lbwlylc`k(kK6OaUe z-P$3J|8@m-0xvk%Y9abQ)c!0w#V!^UOHea1>+~;`MJ^H!r7zvvZ}9y7ixf-A_r*H%)%cwnLDsYAMr_@-ize=ke$r+J~n-{3=XNl9WVm_TAV|4_nwH+suW(2KytI zRWoX4%__azLhN@hHc_Z7^Z)mDWYN1Yd7<@BM8!htKxz;Uvr@av zUYsa@szPf0z=R!M^x|2M$m>m1%*^9Yrk$bV)}(T6y$Ab#`>bLA3UtBft4SyU?DDbV zptZCbWnhQ*Yabfr-U+E-IF3O;88yiR1%7gH=M;MdvDDG+WJ)MC5vf|R)eNDB77HFQ zsVr7)Zl4Z5!hhXkoM5i{R}lXQzM@xKzAYozeNt8J0`e1h-ioeZP)m!iPpYOM_q zU-05_qwfm-YkJdKm6Yw0-eCYm9@N3*WxN6$<0Iwh%}y`h->A5=Fg-IRhf)u^iibwA zuYUGVCke$ukJmt`VG%JPAHTovi^R8pVE%LgXq^vFtHwNJXS|J3gX8iDeweQi7!o048~M5{>jBy}6k z_q6rDlX5kI5hhMSjf^?b$@zHH*BOF#I33=wMsO3V)@I)_&u+V_KAW9$oR(m&-_GY= zY`L%QPW(>W{uzL`dFq+N*0yFID-3!JS89n~aOs2FXBg88xlk!b zaOvK(%aBeL09^kO_7&r;U}m;&+BGmvshW~)HS$y~VtBAzWesZK3Guq7>??1&uZqTY z_HUkdUa%DhSF%2pwS$Q6z0`Sx3X1v_MESpQ=x)@Oo?;pLg--A{vQp1JYp6XtSfIdf zTkE%zYdR^$x}CE535OWJ$>sKXHwrkM zP8dSYkr{c_1>kt&;g`>mOJ3bI!s$0b;&>C}`x1JO%ZJ@E(zeIZSDd@BOG6IhWk!|G zx~V_4A`xpnXEqc(79sl47^I&PYficrkrspJ)2_PnwwUz0BuJP5{2gIj+)z$=S!0Aa z@=U?YG=^IwOQZ23_Gjo&*%*Po3sx4(*kq8n#8tbm>@%cUwx7N5ZQ2jI&VyY>qg_lF zI|1+j(+#@x2s;ZmPLY5?dKkM{{?}BUB$-CCS-N`jta8ybHagKLv$*XC za4-|}yQiecLr_}bdJ(Uj;+j8m(ZdyEUKA4`Z zGP&!lISFd@FS;eo>&Ud^;=cE^AOtadX(Z>-h%2#Z@&7K&V}hcxrh~{%r(4Bf8M0jt z3Rrc;e^KyxHTMogXEVDW(@HaiXLKcBerPhI$&lagwP%c1+z9%=zs=r7woC022ikIn z$ps?+eBZni(&Q5R2`fO<`sS{?xjXms`~Dr@#X|EfQN&`}fX@V=31iHr3WM1nv=>pV zic2%0OShVgS<8pSA$aAvAoOe^EM*?OscdyRau#4Eq{km4XtJpQ_EtBF_X`Uq`afcI zBb{uDul>Mq#>!8{QW9lkBJvElZd-e#4ALD8A{~}%g8-gG>txLt{E_m=ZAW-VAwIte zWIQkV{k`1ZRu8?HW+i>b^_7VPKD?qBW>)2=c?9IH1-j}X6|GbcLSTeDi( zSF0kuJ+H3*OITxiHL*X0bYq|X#S%Kzjebx$xyNWf1+3C~a>Sp;$3z1z+F~3xP3tmh zwVk+unx4y~$bWEtpA6^}cTw8YHE3a5E2OvbxHBhSqxn%SX;UX5R$GI2reQ5;r{+AZ z7n!!O`VQB3KPJXYm^|8fz*u|jsVo<6Wf{42?AlmwYSts&N*huda^rIl=v_#zX}UJ} zU~)t2cSCF+&ze-uAAW6ukZ#mfEdrCex*DtOZl_NJ2TuQe8671~mjtek0g3p3Zq))G zANTh5Vu8rbee?wYw?62WX(@)&Wpoq`|8XtKVeG~y_sx}_x0O2WBJ)XJZ^N<|NiQa^{NyJY$bcUPO#V&Dc!-# zS4f9AH_4_RUTjau))TxWgqUT#=^fxlS$L!m2RJtLI9B7rB@QO_JXz9MWZP#>o z(@1heeQT3A(yhRFvqIFb@jG-n^pI?+jn;L;cOd?io+EG69;d(7NHmlFOdPr6 zQe$C~hdckDYs$^lvCwFQIvaV1V!Kk=*ytq1VS{QV&BwUaEyT_3=mXRCNE8z#01PUCrzL%kAg30UHQz6ZFO* zcx8!BO+&Dd@S@tsf)Um0(*;{WB9(u|ch+OI+U8sKpji>?tV^X<+>BW*;wAf2Q*_%U z?W0TTOAEYS7u_d{_wz4-gTbzP%n`ER5$Jm+jQo1-TdhdT@K06+V_Qz?OB<6Zi$4B< za7wfyZ!=FK0Qy9<;_@{)bsNnUau*F_4~L1GVW#I3ogFE|_C{ww`Ler_;PY*#0B~NS zem*GY4mkLo^aBzj_q}}k;vf8Ofes&U0$QMYQ}e?&l_Wp6Z+4x0K3G2)9)Fa{$bNXS zms1$2fiaxO4;ZuiMz`+kZ6}qatd7Ol;?yyz1h{$@(-Gz9rp1h&KqlhEQhNQnh;tj= zE%5tn-vaK(l|3BsA3|>W$9cR1Cq>wSr(fTY0Q38IazwyYbQ*xEy^f@X6%Us}CsRac zr&t2-J12ftHPs6idkITs$)FIuwQId9)BGTy+L)R?Nf}CFbILPo+91W|nLz2`>dke;6&3_UF0B#e%_uL&Ca#`p;&umamKCThGExX%3yRlxWR{Cl@hbCWpi(=&@x z^9=mjzcub}0!Ato`15?LXO`?Rpa1oc8=$KfIH?RMo0>BFkUD4n|Ss460ArLBuAqTq<#Ci~N@Q1ikz4^x*0Gkz%W*t1F#19#uY z)08$qSIi7Q_ICP>7EQ#OuA|Lw@O8mvM7K{ za41h)vLs6%Q_o31({#NY%=64gNLC^3H$vJxgXs$(IcJYXMz$J0c-pIiU*Q!qUD_(k zyxsH&@`$*H4EJJ23yeg?pji)X1EIQEmegrznM}CJ#W0O5Itz-B>EyEUmcQlsXskBw zpy@APvN>0Y#0(pj9oePGv$UKlX*vw8xLImnhcE+i9m|FL2RB6ElifPT6S9C{w!G)P_3cn_Xrb{1h!EDiOz1e zBa8Pz#A@PsL{tSq|66oS@PvUb+16E4-iK9+2t!zb5>g}u8x)?1Swmyom?#wk?xd?p zT?f5N98O%yuu+Z{w;X<{k%5j2sTcTd$b0M~O(g0)ro-0CoQo&_U2wI4@jo%O+8Dssx^n9CI2CE`Lo5guV^*QR~D;> zZB6gCnnl(FJI%HZXCV0kF^2sTP-p|OML+$nXeA6vKT&KFB!H6HX_L$AL)V+ z(z$%XrDrvBh{(hO<2nLO4N6PaAMGeNO*~b@-Q*GqWSwXd^(4v)tU7ry%u->unmRDjG^L_&o8DW z16;ldp#**eqYEN!Ou9KqDx0 zpEh-^8=+1*GAB+*(R&WDXlAE}DxOJbswk$qlg)RRSio%vL&2~ZtEPBbrOR03E@{Y1 z?rQ?Xg;@jl{ywlOR%txwgIoLrk&EMfxrFMaGTSAlR|ghWBh`sN(YKoC6m{`w+97;S z%rZZp6P}TO%1lvz@}ylB17oBx^%(#t&o=pC4;@;_G?$A;n$`3P1oatO(KYpbq9)9G z2SIJ3_83!v$!+Hp(JtuHog^F# zI>?w{R9Z(c&zuDt!fccb(o9UP`9CB}h3A5K!Q$=V0hdj4Fl4S+bt(F@WINU-oVAVQ zJHUtqRyfRppY=+c;#hyTO^W3n59G!W*@))pVg9+4zbPk~h2kK!{}3}L{#R=zK^k5H zha)oBoEbc><(I9}(Wad&PGHg2UZtBaBWWkpTkkKuq}CgZ76 zgSl(Kn#KV&q?45piz2AG(N+9z#kVWLSuyTy3FxtxF56>}Fv=MS7V{9m``dab%W2O^ zsy?Iqnc4bOt1#deDGD27+pfW-&-{h%!gy~BpfX))DT4q|rP!d^(7~2jhYy1-H7v3v zM@DnP(=xzy-mnqbp-O#TXF>@Zrz21(kZFIHHH}j4{>y65`c&#ixiV2vwNV|OVM(s- ztGS70vdD4M$MTE|2B>PC?QsR^T@WCLDiw&j;Qn=OAf-@t*i}+4I`~?e$ym~AI}oU!&UbpHig=D!vRF~av8S?t)n3aT(n*{7Te)z zfr}jLVn2;g+FDV&ou*cDmFvlrTf~TR1Kp)s&~%6t3tbnxUMJ45AxC&s#oV*@k=DWr z-K)hhI_dkpZZbW9jQxGq`kSKGfuqAKS>;`!XwY9%~83233N zjE5-f&LM(Z2v?+M{zxBiIO@l}aF7j4ZPBT}IXtUbxO%XF1c)aRD zbOD);=OczU{RR9Yy$Z8T9u#IiTo_^?%cIr1Ojgl>mgmo^ z!(DF#6_`&qzITIE0pgiL7brMC<488_OXOH(fG8cc)W1^)xKwRT1LAYxaAfF9PYk=c zIKeE26JRHCtIJq(sIpSm9S)hQ%8fZBjd<+At~Ql*$l?pHDzDmw!|eOxP$%;0B;om3 zS7;<xTHO)^K_W}dFX&=*l0(wQI0^D@J4`yH^n-r$M3S%@Z(gHxPh<6z*k?7B zX+6M#uTrmBQHmqYu+#mMHl;x}#kdP{F|yO?Jr}TVtlH4_z@0IEb$9n+6(Ce|L|{Xs zyD*5+|7|TCqsmScijR@_``FE!28CN9ZQGZQQLglkfX)x#LJ8o=qNcknrds5ouv|CN zT6JW?zBKr%E2j?(tBc=Ti3_wYvwu1@*AKo9+Gg0|U;!z(9^K||rI4Ea3=yHU{%~Qi zo|-?G{Q1YBVkKYeLG$kmX-$H|8s8i3OBy0(2)OJG->`p+!uk5%dLh)*>6nbAou=6xXeYU5vM&v4g#C-FWR|aHak~Fi0ufw z3KoVxqjR@>5+VP+i3T4(^KK}+MZS5{8T^?!c2b>xBRsFtETao5z{w|gr0_ABY2x3x z2(wL0vrw*VoXs_%h6L$k6`{$ia+2I7+By|F3S(i&c%@}K?+Oht>7(0G4 zO4MAyw{=Lmyx^xslEWAc(XVOA;7PSanLyg8H;eMaMW5!eT9O+&usjov^lZ#|y3qa( zMotDt0W@9df9wUKL0!Hev_2-GD}fFBzDk3N+R(}<`!U&;r)C(oL}qfA zYYEhC^p>37pl=>*aW<5OV^a0SowNS9U@*xbB>B|K%4LSTP5_hwb?-$}8M=#YCSzFI zXn)%YdB4b`M=27YLFU*uq{VMB8ZfzXiL%08-Q}+vC=G`?|M(zd^^cf;d)Q6xa{4hg z^$^$*`0{w$=mKQ{fh~8eRER$*Ab*s-`Uxc{=6r>DYTOE0opJhH!jnn(0n#PwY|VuMse@M#)Mwo9>G1u20Ko-)uhqDj?du^X3o zsMGHz$9{CkBy>&$VqDV9C&utFU%J5;28;=&L#71;JAwQrBduDsf7LPhQ@2`NP<796 zOUUa+(jQRp6gEd_2l-g?1x$doUcBJ^#L7%zbXVI7sOIEIMoqXS?h-Fh6?-f#B6$p= z){jb>``f@@qLtY!Q*SFAYcGBz^e7vL#!DO@@eZ9>Kp+_K&z}V+&OGO*=&3LmNiNhk zTa0W{qA^+2rceG+>KR`}$MceO!Tz(Nx`;1tjnn8ULA17_7n+=HBo>p`+i}b~m(7IS zv(;lscUWLs@~J^8{gbBOP)mIaf^4tDC3;hu$_YJ+F=_XX>^04bQbEbvUp-Apzt8<))Jr7zF8^!(Y$q=Q z%obXg{Ew9)@GzDLsHIAI*!Ro|j>)6kLvajD8iEu$K`RRySFHHiI{a7L!E#NxvyKln zRTt7Xi8RQBsCDjO9}yy@H^aF@Ez8RO0c8+9ItAXd^6dwSg!6HRkN3!-ADBjPR z%6I3#wFH6Dx$-nqA?cU?P206IY31~W`lY#Jx6+grC;@!Z{T0x+m&5E&()Fw{yA=ek zL7y;^sxmLCO=hCG82(_j5<~Y|&)GZl zey~(jrb??t8AY&7lH}3TBO|jC92G^57*pL1VHrH8T3={LY=a zi7H*hVD9>P@9fIt`XN73+;|mp1saIBh#^Hri+ON8#P9JhzQmsVp%8lh?`lh*^1=1) z+E6hd?e#3OO0Miawxe0U)!w~cA(^Rd*TFN=F*B+g!HXYsV1QLO{0%a= zy83gbNKhgm(pnc!&!6ARhn4A`525$vNO9=!xYF|zxP~TDf&6wueo@V@e=vKZ{0pt! z3CEMN3%=zULat~z`x|4B^B#0pCEQL)5T#S@s<$@9n5?6%HS(@vWiFE*!mK#YiG99> zYhk(tJhK8@hRjc10+NPSv4$|a+@kGTnr1H>>@cIue_TU=h#<&#=`|sql(s=*q+D8 z9@^TV6K$3SAP*WBI@W=7MSd`#%E<^oBNTGc!_o|xzPo3!F4Q%TEZEhqG9c2F5~VoB zEjlWd{()jB8y>_)=lsryMG>5-)F5?lnUZ`u`azch7^bJQL|2T}?%6qejIJhwtf$;l z3b5oyED!)?9-y+4^s_fl$cSNXy!^gf^T}z|J6ms*1?ethPki-u^PSyu*WK57?dCV2 zGmVrKdl9|KhwXuprdR11w$&f_dX-&i0vfQe&k-&s3_fw~;=CLAv^T z$c3kKS-yp1y*+g@8J!G-ZpZ}LMzS9bfXu&gBs5P|%&jR#gdEZmP`p)XeCz=M#g#h& zFx~BkJibecO=M*E^r>Y9h&{eB&)^1LHClcj-Y@pjJ1*$mmG_D!Td^l+-cy_iP!|Z0 z3f0zYM6O~caB9};u^QGyv&TE3sZYEQuwK}T;RZwHP86!r;#tlqw}#<=DfrVrY80 z*taa%mxR^!Z+Rgi88V*rlb99G8@`5J;j#m|(NY|@Y3MXfAc$9Y{)1R5Q-8yu{nebn zJ5hb(U7>H6FV=-=CvjD%3tUu)%3v|M;&dMxI&i7~^28IGkWNaC5J$Z4i@a#ZmNr|I z%auJY>m1Hwv*c5NRe@T5TLFvJw9Z5*GSK}X=T7(8CnSnYHCz8tnrH?FZ*C$3qCqbFy9=}0xIoj0igQ)OLC8*t%1<;KnS*6-)i0l+FbB=W@Qs~?*jZz zNZDf?5-5{nV~>o$W;sm;7SUhj0qg#JX~s)*qce2?K3T^?RS6sYsuAS3J-Ajxy{RT9 znHpFj)e*j;1;}ACFM;;!!h=&X#7E9sOZct8G~-}M(JWNB*dNjDr`uxf%AI&*Qe&$6 z#pt}FOu08*KH%!exOuH05a+(wioIdSID`OLH&6uEnfinM~z|ASh=l6rB8@u1?r@DpfI*+D-Z@`}efzZ&QGH;=deTo$Ja z#Sxn)n@bY)C&p{aO`J}d*Sil9Y~4h4x%^x;`-HW-9R!gNS}k=%roY1M8B{+zO{TCw zI_PN^Y0fqO{sEF_dcSyH4qwhsEVPNgmjqq2stXCio$!ZA7o5)vU&1AVy^4)fhjeA0 zZJt-iB-D&6jNQB#POw?8Co{b6S*l8yCDr|AL;3T=uLv5l?=F~5apfKXlZrRa$NR8xkPb$s{m}M-+d7C*+DP8ALs{E5p zOP@Ml@UUAihDS&CJKJ#TNnLllC90~hrVVl{reu&35Y)!La$5|N{(MMu=0Ye{6%B1s z?>i@fstuWtK)D4tC@(*=haPYS%qbby;(}B-`XdY3 zl+HE9bwpR3pi4(K;F*k@+n0i_x*l5V)^r{_h}iFGX7WxWS^gFj3J=p>8{VlJnIKhg=9%M&lzx2n zzidvOO?n~MOiBj4K%1~mVf2m;N@k58d7~>!XaMa`R-Ra(#es9NT zXSNtqrXyp+uavkgjZq6uF@J!h&1|oxIArJCjHk6F56?T|jq7o)?U-&mz*RJQ4mZ5j!=a(-UUcyjun@=@(3OE;#&)io04*+05pTF0` z|DRAHu6PlTh$su5o#*Pi9J?8brn=QpH}F2tc!=GP>Ri?);OJ(sy5E|=L3Z~ywop(8 zY6oekoL?M3d*l_~)z^jHaZj?3EUHvmD7B->WZ^pIlyOzU%vg*!Y!RE|ONbI++2t5> zWMz^ussu!2rzS&BX!a{jqLf!L*c8EFlVty7k(Tg8kYpv7T8IdsX;vAe)8cQl`W9Rf| zRLqiy7kQs$mzamQ0}=Aozbs6AL&oJ8kZBS@q1hDv6i0NH7&$Eyt#$2h2FwloNYB*^!eOKvMuRY?HJ8rCh<5T3+Q$Sy?qi0ous-w)Tn%B>{# zLjm-#9}*TP=p^yq6_qGe`5vChvr70TJupw%o@5z>LPZ{uWX6Sqr4o;SsU(kxQd$XS z6XMBHAQ__si?9n}L7^ayuvljb-+^{is6?uY1<26=R+nBz8gua4iod;RVoAKv}&x8t`TzJ2}Xctl!h-P&=iU8&t1@Q(lb`iHk8 zVyHq*;z>xVOfoslOU>jqmK;l`u>t`d4;>wC0EeV}_pte{W3Y9)a0pn(2NOEsmoRTg zs3@v}VR{%`udOlYtR5^AEi%v3zIr)~f2sZ6ekXsF3MDI~s z7@2YUn0%yUvC}M|dD6cehQsIm9?LE-X@L@6a5)Sj6?b_G`F=sdw?eSg83+}M9xY`O z5{9AvQ!D0QU$DhD`k{(Tf{+0FjF7KUMZxUCXc_y*(f4n@@$Nr}KxY_NAF5Mc38s&%Sb%>+)|oE7PUFcCqb4Z%$o z@`b4i1a8*6U@BMKcPyQ^Ho%Of^H9u?^k@94Py1)G%JPDBCtM#y) zG|x$_CV@hs02B&^dIa7z?yj(HmtDA#IBbt*5m$AgiFplgHhD}VHv%YcX!=PD9VJFX8k9-_Rvw6DZ3>eOqJ_^G(@bdb zQVp$8$qA`|<1rq#lZb5jU{=Ca9{|$-M~sU<`a~ySt)Lg3&y~ z#>D_0NFJFR!wh$e>7x99UaqgJWKoD2nL$P4iyB zZBdZzW3tJ%YO=*OJi=tf!0x{5CzCW^G=Sr+T@%lJ1w>@va3B4*qc z$tPoHDHa;fzG=6vlxII-8>%VJQ_WZoNXF|1 zRo!tJ+kw!|!Tn+=w?3`Popf*ALV4MWrq>g008gLPc|tMA&SOA$;XdDb2kip z@OvOKaBp&`lxVt*W2CA01!#9f)M;QEcql|s2b=`g+{|?#@V11UVGfCc0^O;bcr%Pb zj8WA7jsDMfiOV<0u82^AyZz2~XR9ka^FvuBBQ=|5P^t>#?%t_z*pc|5WUM?7g-Ma& zRkC6NYH2^E9hvruh3ybF2fBwVH^|USib8T#ris3!!GgHm&%IZ1mXBxM(qKV- z^3xy~l&qR!kbey!frt7Caa$29o$d3+4I$6otlO)t;Pt2K4tgQo8c%( zFd6a}T8Y#;eXobL_Y7@QXl?WHpWX}uf7s86pCll30;q;){Y()myawF0Xg~AGm$&_f zP}HVlLWz(qZZ5d@6sxs%zt>x{l%v8@n1^j3gzeuc2ErCk_%m3F0Jc#l?z9ky1hrlA zoiD>NDJ&u{G|CzgaiJo=undbfcK@E(#)Cfz^k*Kdfo168;?D9S3|VWLbnJ^7;t+;I z_&H97!-7Jy(uu=PUJ1lvp}u^XJ#q0zE}*wq_aV+aQ};9FU1x~ffbMK>6%;I)@Ul}= zmwEr2%SSi`z84Ery~rIBBN#HavXz~U5uzAg5Edn6Uvf0F%m^h}K0zN_U=#IVG-ayu z0so7s<1&SWlR&O2f$nqwZ~}+)FTrw{QqChhqn^Q6rCFc^W8ydpA%gf|C^$}3_)*hG(3wJv-7$1P2i)C#1b26ttvz)NzW)X;T$8!Nuq$~EOFJG;QXkAK#Ldkd-_)6*oJm{77k+=EQ4I=O^ z8Fx~taOmg7LVdEOG8(r%rXrRb4H-T|@$Mhe@4`b496C~?$`pcOaHSl|(+b`HlsGVI{hiDH;;THqYDWO^MIl~s5>Vwb^I%O3z?wE( z$NEr8VQp>AoFu||b&>;~oZ z-R02$T=KJP3XCVCr{ls7wp`Vo#u@>6+UG7l^do0(AA*aWUJ@t?GBy{dnhk3?4@(+$f1f3Afks)qOoJ{CJcHh@=fRk~fi7oP-*vxRYjIN$aTm!PEkCpP zQ6W&bipJ{fFbh@N>SPM5w37V76Av)WZCXuR`q_I{5$5vh#)1h3i1{&T0a-VN5dpYD zWYS659z}tfoeQcB6w_gf4X#n-ev9TBW^PKHUWz*yY)8Ls6rH+BJXb>^IGIO#Ofp{oQ z^4kjg7NRkh-_TeKDGq?rpyq0dFD3c$lihF>!yqzeK>kcXQ}=KR150Qc}30Ef!5Bd|_qvP>>5y{Ki% zsqA~^>be2$?zDu|#y9YVA1Zhe7-9(S?$*}m-;odQ?i@O>a);hG{q2OpOSY{HInxVS zF@=z_Tc+dNVvap&|0n}s**9h~)3^slu1D+7l9285bkC)$C_$gF*n@d&2kAV_eGiye ziCf~b48OzEuG8q=d(GSr$h`NYjtdcd0iflKl zSCeUEhvJ4;N}py`Hx4>lb#8&r+o31ufJF^Tdkg&`O2%#^Qh2J5+{KX|G|U&u@ooj2 z9PXbz)GSkc?6op9mT5r=;^zM{olcs^CiLC7L(nt?0p)z`?t|@6Y`p1Lb>owXBeRkp zZ0!qYctb9z8Ei(im|@P06gkd(DV57I&y>_Q)8#Roz%Zsq2@;2foEJZuz9S!bGU`N^9W0J3!WK(LXru5m zGzY@qK}B+mfXn2GkF;=`4&28=*Oe9Ukwg2aB0X}65}*+Wz()R~X&uRKPnEHbwA~S| zs&Xo+7Ua(_Z86p6Fu6h)4ns5^(~iAN{mTFVA3lgEf4}~!-4KJ9z?t5#Q4+U7(x`z& zVR{Mnw)ez~8HPiQLkU(KYUpYjk`RNJ+L!=4O6!HBKawq zz-gLfHG#A?3Fc6!B`_E8MfQA`fDpQvsKvIcVo3=D$u*+o^&E4Qv74)~+bi~N{E9*QRsdxamm%hfvUy}Uo+6v=*4IG{D0^XiL`nuMM9qxl zb_MDSI(LY642*w8u*byIz;UBn!q?g%p=YXDV1)3_zit9 zi@b>&jiH}TU@C@*Jh>o&!fLi7KnCLoC2*g^Ig?gH(yUjhOrBG9<|w z=4#g%Vea)o7pX7@+CMqraT^(`RA36jv_UMG1O0fK2FZ(NVJM)#g=E6hN!zI$?y501hFH z8Ljtvl%1i-$D9PrP=(_;;BCm*1FF+%Zfz>}mV~K+#Qq{xsOr@UhkC)pFBPdzpsoj8 zqz7OZvj}1V`{0Gpjlvt?g)+*gSay_2N-Hc^EkZp6;a(3z?*}lQc@r=R#&kawr>QWw zhG2WIr+N<7`;!f}I7ZOrgHA~kp{Om4fWN;e=?DeIKkJ1K*bz__Vlt8pg~R4D|FWrs zAlhr@0w=;>Ag0Sq7talvfr&eZzy-p1AD|g=$B^w(>|TYU=D$m?t2(|U>yhEUpGqVv1gL(DDv42 zHD6w7R$uN`$jtaSR~K`PzIlIsBI0Zgx?M&zD)6&VKFrRJ-G`aiVA9&C%OOafw||9E zI3sE1e>{PdC9zy|TsbR@EZLM3rcq$b_7Dn41pD9>v?WhB>~mJa+{$G9r%3strzP2e ziiH3r2=JaqH*!yt+FH%cpH3t>5Qk~*X^x`g8!l$<+-t^fYnE1&CO{ccNI>RvO9P#-P1`r3INw)3(6r#sOGdajkdl29 z!9nXz4e3EQb!j3&WLo)ZVgx^4NgYBgv-%4cvL((wK8sjRosWWHy<)KF zUM*9~bCxA-VRrMmLp@LwOyusrFqNKLOy`l_)Tad|@&Ck-z!KqX*20jvwz- z!eSf{=<*@T;cAE!O3=XE#JYi)r3qNw)CE%pSHPMmg`b_M)mh3~MY2}K1PBqj=2CX* z?JqRXLbzsv7oWmn_R6U`PTY9H;#tW-{HGTFfdNqJii{fLqq@ix`jJ-&n$puMzF_Wb zvjl!*iX(=U!G4&A#J~s3DjjXPreW!gdGJJ%g=V5~PuU`Mg?usR4VDe2W>hAjvcN}~ZW%g1W^g7;>lSBR7w!?30t&dI z*)|K`Ei%;2Bq*bvzM9z`9SaZ3?&ImKP@KmUZ&-#3isk!b8EK8ydmp@)7x^}*QY9?{ zxUh^jn#G#pwH!QCe#;EtDZwo4yihpTU_acMA`i6QxkB?odWeIC0#|gfu!LWEwi-#} zZlTV}LOG%}JzALUd;==V6y>F!@((0CFXg7C+;m?(rS2tr&m)Bbbl)?D#;%$jDzwd_ zxz1AO>-80vdaYXOOT)`mL%390Q1oabE=bGE2M`sczx?x3Kvs$b673q|059556wu*+ z17d)P2e1?YmLkAX1XzjyQUuWL@E6|xWtfnAZvBEqEv`$nZp~#oGvhX5ALQ90Jaf}t zM3#^nL#Z4=(KwUd5qcj=L;*pcCSri*y?K8@Kmq&4BmsqY9a%tm^EQ(P%r?FOc|an( zln9m*!BQevN(4$G(9^izLnhEeF5Fu(P|WH~}xI`wOP8}syvVA%hw3Gf5VkwunQ^?4L@?R5RSu4aH3!KLTu-)nRgx^I$aXfLs zx(p`GIhUUN%3dpV+F*tB3|af(C-;W?9KhnFG%b;;fYyFovuHlJ7m$iN^4sU_6+Y;p<9%+`j;j4Is%nHSuY zPWT#@K{5P^EFMOZ8y3UR#wQ0&eY>zZEjrC;PG4l6wnKH|$^af!9M4o$*xjdWozPNIn1c*|P$LXTYix*nbPiE7h zW$Z5B!qQzU?-FgnMTMP43E$^AQ$>lGXls-mtktN?d;qU**a;-f92NOmZx!eP_`dgj zU)$Zz%$H0I;IfSy(^*82heDB?Uz}a+xOB{1kiuD!$>jki0ZTB(fvIHN9dPtn2(%78 zRXN0q`5vIPEmoXpT=5u{P3uH*uq3oeWj;Nbj_GFs^~%nTT{fIlkUql9D$4nSEa z!d*+&D^aOPtD8>wOg zD;AnY8|_bpWIlm(Hqq|!VGl)M6jcX%i5tW8fY(!J4howEFVih*V7gKH$m@8Ck2^k` z>x8pVjO|uhq}^^xf)1xMt7#5mci#WUyw~aXJG~XUQMLJ;<;i2UNi4I;3M>JOrmTJH z+h6lQPRAqOA4*;sSs{p|m6J$4OHUUv#|g`WiuwA)YwX~Tk_37rOykFhi5=L^8rWDn`1MjHoUfPaC^1Q%V@-cRA*q5P+1) zJm=15iZ>}_BX#h|01$FIB^E>W8p61hM$yWhK!7~v3yIR?wk|l;zp~>TJ9P!o6`Dmp zAc;E~1s?stjOFxj0a^ldK51ChV;=g&;LugAp&41Uxg*hh zHZKHqv9G3uomMNyWL8$dKbQ-G$$wN7DZvuh;to(mIi1NI)WYfT3?pKFDHai6FbYC; z>=b&|Pn}ja?X?O8Qh*JUs*YrswL=uG5t+(FYBTxGh;Y*p8cUIT`f>t+z?KE;R;k%L z;1{MJ=Me@(4sgcMDDz6=3yJ-WLo?cACtjwfm0H>~Wvs>t6C!nOVluFHx#Tw1V-oSm zcX}7N6|o!L0A(t=vI5q#xB(wx5=4BEXoDMy%0Y(H7ZF*MOK}t9gH!e{qK>R^MI|pT zJ045T2wyMyh*dTXKhxbZAgObfxWc9*)fRe>|KxfWc3%0r1aZ{)?~eSttrR5LH# z5bU8T2&kL&z|%J?agIgVp6Nv@IvB)1(xDlE|2#N8Jhyg^Ahk+_9+wTQ&JE;JzULwr zljqlx)!R}hEqXrv= z6tTHQ`boNz{wSI}&|8!fd4?fN&@h^X7m%dEzYnB?wto}3)4&5$Hws4dv;*dWD?85< zn!RJGd-Z%(dJJ<%)7tZ`h6=#*z6>&V9Yp>BNWtMkM!I9%7ZTUZ<5HHDjLwjO7Gv1E zhLO)x1C6~{B+zr|1N_ z*U*<)IhlC?kI)EY=3jLgn~B~i!mv%Cca7QYE+@%~v~Q5Hj& z4mc*dzUS&j-s*Q+D=m6FWFg|xxW5D$fT;9|W};DHDKO;$5jA@eDi)}_BKSFWBQ^(G z^<%JNNM^CT4b%=o^393_-&Wum{`=xg7R+IIvq{Ou(yRrT-ScxdJ~K-+k&;DyB5t-; zR5pKcCnvR)(7R?_iYGrPm<964(P>$~3!(Wh>Jhi4?NoCpl1nS9$)!*rF{{{3KwzJSP!^BPg z(z|zC+0K0mn39g`V5;O_3u0bc;m0|44>A9~1#;0FM`32L=kTOFnd$mHS_A%*!F2>NV0 zW0I1FhP>fn@9*xUUO0_9OhrwmOy(!ffJ}l&np!bgmlNMcU5z5(?ygf~06KW?b)*c< zte{#v8hBA4@H=+RA#BR7g2>mU_wRd~AP%96A@`PsTC8_1dObUtIMcm*0oAfwShh@i2m9$Rfpx5qbIV zFkX`-cpB5IAnK+90UP4K{Y@6Z6ID2wMIG@|bmR~cHwZB|YM`Vk+@#7vztT!=I#HPD z`7TM-JK_lY$-TkAOI$qZuwK7n=j<7gVPxc#+0^VL!Ta-*b z&c__c=(Th*t9P_6kLccFfAT=@^c!U_L|Y^8&zS}6Jir0bScy2vi}5lO<}(|*K}$3f zjO2Aa2F@w+;TcMZlUrCUV1PuA?!gHGr|%ArhG*~2FJG>%>s+fTyUMT_Di-on*oWS) zok*Z`=cIQJ)vI(PkLdAP22BbX-xt|{z$wEM6PL^=KNOjh=sWS_&XMSN6C}rbRUkE3 zRb-Sy!iL)zmlNdrKf0kCc`zv*8y-~on1HE!0|?8J3r6r0mv*z%J5$b>qg! zuieT44>d~46;8)Ki0{@mj1|6`B0MPCy4rjg?X%#3vz_BJ2h>`hXYUS~rBa4GOpnrd zd_v&3aojO)MQEv)Mb&xD&3rA!oFk9b?3_ckwaGlDPT>NRXG;cKEh*Wtru|_-OGYSR zNBdPtDX`76H|7{H|FgB-0Vil2MCrYQ662U$feDO3Uucb)BSXSZodvRBt0h=4Js!#e z^jH~S0wWLV0>a8eKLR2E%zfne^z{IorA;$a4t`|v|N7UzGDaYRZDGhIC#FJVx4Hmi zAgV{-t762>onbN!B4$|#N**2JL-tx5&j*+Po1FiKTrBC3IBe8L`T4KE-P`MB&VO6I z7hB8o-%~udx83hr;I*)S6?zDkkcqC97Wl3!7z)=UrR<59Jamccusy)W!mKI?&dwOj zRUX8D@Dg;Gb)$NpAVk<}0lLlEm*O|V4e{pqoiDUY&ja zK=HYTH{kB>*Oq=5u?&)4?~g<0!cDxG>T*Ts;W|WRIY%ch{t4d5Iu2LRI2*i}q&UwE zNvqM`trPT5dOjC*zJQ31ZbZP=hFF{VSy3O?)K%!7m=+;(@Vw7v;&}wNY ztqac9p=V!St3zLa4J(>r8m^4+W2||{-*^%yDbb2+QOF> z?DfQlsr#9|ly8iUUkpQnHQOD#Z;*0-ie~Mt zEyp)<%saj3^Yg>@cSxGL&Vhr>w*OK0KUuSp1s56NjD145%@ zeC!)q%CgElkhN`(u2Z`(xN-_n1+1=#kcg3;_9wRsaqrTa;0#^55yYC2za|tjXtn~E z4PRXonprIc$8Us>g-_iDkDpQ-seUkxytcV535UK|~~KR>?w=kW0OV*kgJ zqv7%C@#XRU$#DPh?eXdG;QVO+^7!59hFRHWz_C#dpTj%bzqt72-TC2g`0DuNs8JOr z-E-p&mhReqeenfc?d`zG17E0kuHce*k?pi8^@IdOVj0$1{SS?09Kd-*FzIypQ?9=G}N3$Sp(nk6H&(?Nt zZzp^Iv-M(UdH?ei&k8u_Gr-s|c$|r$*ms5r90!;rH^!zRS;;s=S8mwxTtY(F;Wb4x zjB`;>Rg04#A|v4Z8}2#_0g!CS*y>imFDzaS1{bq9MhOv;uY4Q5kW{rjQ*4A*539%PC#Z-UJThai;rS>*#YV4IFg$e8%WHt-@muHwkH{U)iMT z9I$Fk3{zJc?o`DO17841W|0T>w%MP-^kOy|1)qV_cJhkp?su)!4??E1n}y*GphL$K zp}TR+@b0vJfgCm~PiPBFQ3S4_=gu%>u6ep^Bex$7U?}nhGWT;iO9Kqng-4LEHD!t7 zW9V&kTE`=ha61_ug-CxO7pnj` zYN2rV(2`MQr5p>d=UvgM%>R~*;tLCG;d&4_|Ly>1=q$Q~vI|_r?GeGJ3Q^C;q_R#; zvsg!@!^`l83Ip{5Sdlh^^a-IX)5RY-TS_Yc`@LIruP-4c_vkqr@+ty2Z9-cqr3A&} z-qbLF6f6VoRLI?OQf?=L1ThO!#JPYEBPJ-YXbGC-UK$Qc#DJS#F^4NIM3?U^!n)5w z?p#pHEE6m&pi|4>9vs~&u7IXf6p8pD#mMvT9PE5;?ugpP9OlB?Bq{*z=uB6}JO|l{q!b`5YlarDADmgIf{&lp0*qio9Bh z6qTv~<-V{g+8w+E>mfqdvluw@z5^Vt=yNO^u5LTNQMDm&XcJquCExI?WAT#RAv!iE zx!l&Xp16^}AI1~+bT$P$y^Xp$Ml}-Wkj#=u22D&*oCNIDqi&VhcAQ+fUI{VT=d_Bk z*RGm}*!{C(Q3`k7xEMIRu5L}xx&9st$8tknuUR%H=a<)`N9TTtNZ*7^(vPcTg^Cgd z#9}uDRTLAE8-i%8^?n1aXS4`^e+l}X{(qF~5yZb68G>1x02xENMesDKUD8@+Sn^{E zKNA>nYl4Rz#5Z)3&;w`)&X5oGMVM}wlnm=6c8i9zf=F5v@a}wzb^!prn}y#6NyyEu{-tgfuuWtz%ArhL!40S{_5EJMVYT@^G^g zR~0UssH#w-Ng9k)kgfm&Ot|x1C0_~f1uL4;7yiq#G1=GqGD36${;~iVK zZIGU51wFVxy3VetxqTr;Xk$aD0JEG-8W72w?JCoAeWZkC)Bz8G^JxNENIIT6O3Q&I zZqkMLJ8@-17ZJIv#A1pEm*Z;50}WPAs(-~o-ws~#s4ePiBX197&}4TW0J?Ri-s(#h z3}zv3AFD6NSHwg2+9)*=ujf#4KUDP^wp%L(Iy3jA^DPVGLT;WfM|EW%E-Ln|bD zI^$O3g^jTyq3<$uSSXR=Vq=8HA?p+mR5lb7GGCBIey>+BbZZra5ljMai0H2x<mVTV5PE1@zjOpiMPb@LEkc*iFz{hS0&+7ft}u((wdzpv3|e>M4io4H z*gkS1=KvdCIunKzL3>ddt#Zir$st|5?ihCPcF9aEsPhS4`R=@M))mxZQaQRpb9;x1=dZ& zKe0IH`7nMUh%kY!-%(Lq%ouoD@7ryg==?kL&253{&DXYBTRso>G_wDUk>5D@ zN0t3&e`jlVH*5dd-`QHm|9Fz8KtrmZQwj6lWto5-OeK3q3+L<5cl;MNj<*#dJdC^& z^x%|DG$Gs@9Q|-*OC4u7D9D@h@N4QBK~wtf!YIM~HBHn0l7iOj`b@+^j%_Z8;ce=* z-l>RKU_jThu+i^&QnqGLb!YVmi~JRLcP;h9pn_4P3_m{-`L6^lkqqk|JXDTJk$ zp#Trl-vv;Wa;6YwTEa-o%M~C-(qKHWl2~uHjK&Ps1bvDq*6OR!vq()IE!6gCsWx{= zloF1#Kj)_nvDgc3)H}L$gtdKcN|ETs<(fcURtHa9sT<-+YPl<|0>xccOBRfRcfP7CBh2RtFJj?HyAF!S6REXD>Z&4H`y;SBa*R5d9ro1vt&91CCzlu+IVS#RPxG*Ky(B%m}I z*)DsH@BjTcoYbJ~%915S}<9(Qm?%eFCS=RLCk=Pg#rrId}f{Hz0t4_Gi$A#7j$C!xlN*$mWV z;%=QUmUH5Iqi1pek`VhPK&m<$Wl4S`BQY0+i)A= z7hm>WsK(OCZ#4FT-@xXVo%qJVhK~>@p>=92XA(=7=x6m)FC%l6^CI$>bxP;A=7%_2 znVWf);V?aGV6V-gJmfYn@vC>pFiSVu(z6W}Y=~yD!tL9J$fBF}Juqv$K^s8Ma=UV) zEh#ti9&J0a%m6u`#}P{4en=P4%~^s(3XvQh$FEP{ogWSNPtK3_5C1tlJAZd@ba8QX zn8S%#P8H!Ip}jr2eDf|}_vHBE(#aG(+5hqA#MbkI4)L6(COxbHPxSIXyDGkGuI=H^ z`zP;@vKZXnimkX)xQi;%Yl|x3t+28whM}8y6T>Uc#DFz>wr?c{<@qYobWZ+j zj>~lrUxxT+m;|GnJSG=E9iI&^PcDW(ADth+`sc#4z7z#731VVt`{4Ze>~eu73mXiB zIgB92X9>DeL^EDKlzb{%q--?EHEF(m_%CLc~#)!MnYxT@p)Ol8)JCAhfQ>IkLLdK2)en7gr{xP3p_wt{4Ta`8(Wxy58K zV{O%wixxI0SV$sLu$J=BQXVRpg-}=J*dB|^aAh~V<~h1H&-o(k?5WoZ zBdT-8_tWeO{^)wwpWMV}K>|p?LISWw&x;5H@sl9BHnSMyKWAF?GPbq#uEE!BSt~4d z#!V#d}9y-kgG8hnoJ(=!K;u2(R1hx@0e z?=G2!;&Dc$k*ck(d1Xb)cbnE=si0db=#~n)h6=josy1UEc^Y+FR+MV2ax0L-5X{@uUYQ;RB+ODossMQ#&G*Dw>ZQs%>bgA!KdWD+M8Ys(BR3A!Pwsal^OXtyV zN6q!H?xW51TowlM09vjJi1+fcZLWB#1FZ_BrG9CtUs~#y9>4(1SbYy-1!gk;r_xPj zB$ox0Q+C_3LXorrFv(n-dU=5oh&5N@RFHV7!uj*6a2_jGmSv&G;cOg43oCC#^`+`& zsk*UFs;5-k$X1`Z+D4-JFiIN<&8D&`q0zn7H4WfCgg!^Xe;RF0adQu+%MoqdTazPk z{g#Y$B+k!BN25@BcrUE4s1sT$VU|jmr4r@=lrV}tJctS=<=Ic8fH7o_#_AWF#B$%G zd{8M4=T)?biLscPMG5e! zlEqM+bPgtNLOQ2fX6Xk~k6`AWtYDh+K36)Mntxs8@7LsR?KrL~Hb|MDVb%mC?zn(O&)uXO z23L~4jv?$A(=?`B023H_P@!DXuTH)P)hGPaU)MHht85SPVX_HUwUUFESraI`s&%yp zdN2a(C;{se7t`~PsNwvEQRHmoUXOwhIvbWVU<$rX8L>pRV5;Ou-%aaxGxfWv-nYZB z%2VAbC}O4g0xt@Ni7%bEZfqT}^=o*88F#`cA^TxGaVzQ8S&&|pDxIn>Wis;0Qmgux zqN!HiQOg`#r`-Cp3B0ob9IhSU(+k|*cE9r`(7UF!y;+h_{+`wVc!Fl353V4XxRDPj zS9P&aFo5s6IYM(b-{u7xN^BaUS!$bWKerp^#g*n8&%>f}&)FZDi3-(3N#8&CG)Y0% zn;nOIlt&CIgss9#++++1{jArE6GV`Q z!U4EEI4f>-j4%#jt>HHD;-bd?qxT;>_ite4H*MhkA#Xr4J(De+y*nHppQ$?5yXjE? zLqB&f&EE0>#Ka{tHUbLN*SD6iw^H8I=XneN=d+tAZtq(hzlXA-P@InM>sGQIpI%(< zpPUTOPTs#hKD{U>mnYS9r4%cpfRzBc33H9R`YDOCLtvc{Z!gatW(n77**>7Gie)`! zR$>ic+1f zHtk!sH&ystWN&K=)^9Wa6Nhn+>>{& zbGERx;D%^y=Yt%sl?&DcUB+bv^z(N=A7325J3T&qy;u|4k*Lm9mA1eB>^`r2IF$(J z@stL~4B6Ep=LLotGpy1OE448)4dG!lryUkNa^3vGmxSE+Z8h*l_I>_tOMDpE8KsGU zMg`99GrZosS+;_x5!j)qi?v&?;=(YRH#j321Z_wHez|ULZYU-JirkR%T=`4e&_=GI z*-hyZCvX&eHqUwVP1>F)v08aUU9Z!&o6vSDczyF-ud-r!I@`XdwCM|j5%g}n5WYd^ z8mHY7Z;fU&&f1ao8w+*s0nG<<(51l(S0;!yB>Ej=E7$^Qv~&7rgnW1bL+D(cV|Sfl zyRKu~wr$&KY&TY8Ck*1ul=dIgdNr9UZFuyz%XxSMum||iTqs2L~m1YoslLs@;TH?m2qLlf&<;x8{_5c1Df!@sN?*U ztlJL!^50_5hkwLRV9s>+=1I$6Z|V%t(V?t7>=-3@V@t(H3zdy&Sy-+v7-%?F#xEV7 z1kuPj3Xd&J#mrEEaRhobmCIsi-C-K38x{}Rs)iQw*-C|^Q)Q5S3di0S_!dO(C@R6W z1+#W9-~LLZPAsk&YU>6W#8m6HDo!#{f2*Nt46SJFx7jc-uiWC<;`$xS{2Ph+$CHZ6 z#j~8hy;YT(LwQ5J5Dj!{^8<~6MCqY)%aU}6+WcM5g2Yu$*}gMCm;Em>CopDXk8KS_=eej&u~QG-k+-Oij%N`%0T= z3;oF;*7-7gjr!U%`tn*#z~>>pNED$DW@d9a9Rl39V4IDHuQR>A@q^NtAc7?bcpk>7 zXp&rj#=$t-;c)-6duwY3<*jYy+!C~T_VDz-c3xUG0SzCxbG{q3|E$BKa^6e5u3H=P z-egE7=Xu&)g$tebYvDV5(B3Lo8b6T#K^hDBW#<;7<R7E#Y?t7rU zKo)d6w7ZbKr@;m;d0JZpC+~Qbp3z)_+PHpg=e|!NX8vBu zgDBhi?fx8$ZNvJ9hDq=>KCE~N_jKWb1`Qnsk_| zLM>q=b5dzED8$J^#bpV_+PIfS)D~LPs9qrmqCQ8Y&IcP&Hugr>A_%X#jJ$>lo zz?w_YmIl*n)eOm=xSGiS(>bWQpx$U%g@T`IW%$JNyQD47hJ0ZV^3DGZCKm)#zEgMN z6`=p)?+bDCf*7=gRp}79W3x7t9Ip%C*V`GaJ@-M|CnG@Bt1=R!0f+e&%mzh=;M7N( zTJH_VYXc*5@AQ}#WY)Cu;NNFWDNaWAitMS}9|3P!5W#(#8>OQ0%|`iv0waVK=dwgx zKQCRMjKP=pw;JzWsh>jRxm!-{v1@Y-Ew`i}`(g~u>`IbQ_BBX@w{_!t#yBMQ8{m4_ zQybLT*8OJI4iX(UUo9uJJwmW2t`=-+40#)*|H0gfiwq4bkIV?o_daM}g4Z#Rm5H)q zMTJ-cl4%oNItSEc7=a8rSs&BSzkcKtzX)S7ufYUz?C3SMjwmgUn%(XeAWvj@!Q#X2 zaiZ50tL6MIWa4dm^Wp{I<>ckn{`qUKotg8bx0_igN+;Z@1;WqdyII5CtnU1ZHD?v_ z>eP>XMzF+;n05uH26EfltW@Y4mw%v+srE3#@92gwD@VEtreX;)M^{0Q zG1d-F34K{hqfe$n#1@{j;`W~+EQanVZb$wrw2=rfQylw=0>^JU!tsz<`U3TH#hV9v zjr=WnGC#yehDa#KYMt?xd4|hcG?rj`a*i`kK^!!bz!E;iv|Y(JB|v!SP2fIX-}uvW z)w!x}ov|3(U@(>a7`KU9U~3Q-b_$AgV<%FVI1ZT)$IGOHRY;Lc!k-WLOC_iQiF2u1 z{A9{xdpUL?Ww02z*ypT|G2<7pV$+M4aAz42BE#ZQRGj zQ4`z0L|+whKunU;?)+rasJW=p0l;c=CBH+BEjV*$Eh3QN10Y%s+74)FY6kfwA3rW1 z89l@kQk0zq=5D8`qO#x~^L}#afiO>A$%lZ<#ahc2VA$uqy)8LLAi?hL!xxY|N!j91 z=;_5E80L;iF!7F!H6WsAznAT!H0#OsQ^?g7F2qs;qge)xUX$h|9!Tgx5qH3akj+ju zXvMzovRW$o$DnQ2<|9;H4e3~h+cqhLOWObtI`!jD7}-gkol}YeuMr? z=3Ug2P(R?NJQ(D3ZU4aW-rKuiFac1x+eiRqRG>65t&Z@VhJKrb+ZxAzB20zqU2wDW z4uAItC!sNxroI?)IkGU?aHV0$YF4qb`z;18C2Fg6SEIB!Ml9LuPaQn)YY?pi&+|ar zK$ks|&4qH_R!b@W{ZPA)&UYN6vLdmS=b_7|Sg(g#A$N!+SDD6+FEV#;yGY!I?e7Aa zKwD+`gWm5n^t6rHrL;dBMHp1_JEr?AtJ+EBmAmu977=S^bwapXLlS}%c`+KujMp!k zsODt^O|Lcw=}R!qT=pIRY8d42TQ}jtX2iAz#j=>@Zc@Tg{#NKc=}btz{MZnM9h7IU;UZe zSlD*`qzyv;OnzEryd4tzbt%jzdp7U}eKLhc`8e2h9g)oB?Li4H)g0`}+9)0N2ut~H zF~N^bgGa-?+smM@h%yoJHNl`@hK>i|0r7qeR(AVMweC}iRp%7OG{|3Vlt%*>(i|F$ z^Sp7w+YLo77W7_+FB*X)i^ncJ5#l_ULHH{Z(KU_oY!J=m2xfzF=X?7+@9*khZoFeF&HOlOsI7jrL9a+u3;p5_D}0rI*`ib~F!pB3ROAwQx_ zgLl2r+t$jxYdVao20TjH$qhQgov+hRj3>f1pg)z}Jzbm4?uJrOH`QzvWjxGmk4%w)Y&|JQ=qS}yH9|hv*~EYs7a;1+VFP)X<2cDWeLpAYwM7$t=&_0By1)6^oIM@?!;GseV8k}_X4Sq7GW0KeHwIeXCCs60Q-pBRKGRNoBf>7JbGpi@H zvxC+yXU8+ZMCs1V6RdUcoOGJ0?`Kf&Gaa_k4(@FC+R6(xS+?O7ZpcfB;REkEnRj%| zA+*|WI~Y~yPeL99o4FQi_T(5sBrM5VClOW15z^7j9g;g^sBzj-**CFhT?2#M~hSj&L2MlKQ!Sbhl?*|+mZSQcxC9xw*)R6TEBwA zM-l_)-##ges#K}V3HBbpY7ZZo4kmv>zX@3kr4$rcJ5-7Hf~(@fm?!bWQ^mkJcDspW zUk`91XS6kRN3;zy8od`xaVII`u~}O+&aBR}jx|jwS@%~NR=BQ?zM#jCL%lUKI?q*a zJ68;0{-R*Trl;Xp8bDi@U5yJ};C>L(r{7gLd2a&~VloqEmMu46vLCIq9&fi%v~9JA zCCo%C_>&|SNd%iTgocpvWOi&Qo~osAWI#KRF#H7bhH|Rp-MI{EOX8}v)8SC}8g=#z zCvD0B8trRhzdO+)PVZo}UlZf35lj-E!IE{vDN48@`B`@fZAcDsdibgL0%+@PiUqw< z%?f)d@xvmx_oD7cyeX|hDntwA8`azIqF;pk+v?=M)otLa zhnrVAXL`$9Bg7R-*EBjAxy`cC?p#nZPS<~0x|ynFHNF&@^|yKP%RByj>#b-9emZsb z<~@M6);A5#K=}uHa{|o1WXk0#qmuAa!2Lp7-p&t1F;psOE^(tMsNCXg2&g7gM;Pht zKe0q%aYGm(lcL?R6rC;g6%@lTcrWY-6!=Hx;)2swaq~WLie4DDw28{Md&k z29b-=e)b-+dvnuhU3f|~XopHu>aG~QDK79mWjc@0Du{SQhGpA}De9ET9AE%tV=y)! zwg&Y?DM$5ha$PnJGMjk#@K%Z#K{tz~Zak=HMX`G9bmC?$lWk3EPkPxZ69b=1yXEuk zR?_hw(93yu(SLtJ$jP^7@_qD~zFX>$Xr|DPYbbc>?2IMz3=?Zgt&#qWMK9G>NPg3$ z_EthRZ0Bn7hi`<|rCe<*+tAn7+FM~MCs)IWA+NB2TuSf!L)(-XqYEAD9L&~FR+D-A z#`%r@T)A+LG9*7O{YF4 z%sEVLKHlGg-MRXnl^P*`p@aj z!FVP^d+?$ehD0CHBd3|^N{Hj~cht&vl>U#1KELS|mvI3FZU}Cj@f2poJapDsPrz9D z_=}3Ju-4j})h~t*(DHW{(8|@!%-#DFmWs^9c;@-4(*xm@$i=d2JwXU3++9Vujy zh|r?iy-LW8Bc*>4iOyRVdIzT!yPD9u3caj@`)i|>pb4w*zY!wK+lhZX69L*%WF`j-0qOQMNLE6LBbMQkjmo+jGPEgHm$n+!Bd9+PB??) z4}Rd_>QUYfDNU$|4x1hQNiYuBCwpZ|Ugd6wsz?Mqz8dI#fF75Z|9tMt=a1m84maEU z8%Lu(`4^Xo;nRu0?nWZ(u3@7lrAsgzG_|CZ#h@7O5-b$SK9t)I7>wUSeItpc zIB{{Q-(Fwab=bHSz?3l4lEjc`hK-`t&xX+;u^|PKMTgr65fxgTVa!yAz*YO}Qukw_ zNFKD4o6wUOat!tcFF`4z2T86xVa{obTI12&kM>%jVb=G2Q^TimhQiFWuZ|L7k2?An zcQKR*HwpK7?<s_o$Ns^yE-OE35Z~Q>u#VJU5o># zU1B1ktwrtsA<%c0mkr5qQ}BEj@><^t(}iaGaHQNS7Dkq7Lne{_gZy&isXMg*3`ml9 zb2?>(rEWah3)>*xdjh+^?o1#xh$dss0se75#fz%6PhO1wcf!vsB?&}x!+APFo)_#m z6-=+H?hbG}W_e5|qZv7$b-0AAIy1FT&kTPL(N0|DbO*%j>b~=%$n4ee5m-CtbdD>j zd3}I!Leqft`S`l{L5bY2`}TgSjLzH8(9+{87n~15b%|#NF2N5-F-m9P3zE8T^3ADF z{(z(vA7(jfHp~9&m{FtZXh0#ALZJAH?e6|%wp(;A5l=*hI1;abZ zv?(#@AK43BYl>5{TXQ>MqmfP33gmh>^W0O_H^-+}lTIDIsW`{JtLBQuCFEb~xis&2 z5fsSJyeB1KRf+Ls6rr@r?K4vN1`pq5KBr5Pon(VhVX|29GH%OJZO!p}X^_ro^t z_pmD`15XZ?cSZ2%+IH{4BM`M&~c;2bKakam|fY9)sA?6FksL8 zC>ZUWF~_2cWhU-4LBEm%#cq0!fF3>jPG5ujz*h}wTUk$td@h;-ZITz;@c7w~AP}jAt z_v7cUN82%w<=CkV517^`=%c$Dv^_JU4tnECuzGHX+4(FKmGENkutgGuq^k4+O$>FIl7l4 zZZ(^lT2BS@hxLs)3C4jtM!03^1As9Vf)-yW3P#C0R1fX#oz3~O3+L-dH^4H1aX*Jn zH7Ba5T8)~DuA8X;@nNYv%?G+EivwJFMP zYdP?VvJlhMmVtgok1(d&`74qx-26 zVgvSi2FBwdirTkzb@k|o7>I6FBjQjD@BZPX=Is`b=7SxYXuX|eVdQu=`)Jz?n(dYy z|GHZFCryHSgh0a6ZaE2{V%q!k&YmRZKXfSnvDc1)9!f^p3P@K&ZnLX&2 z$wi&JBezLKXA%UHgv**?X_+s zh8Jlhp>P+D3F5pK>}z1L9o=)TtQ$q1^m6cbg6nWTV^E1Nw*gAhI?y@*PtVXbr@cEZ zw|VAHPU?||zcFJg`8z$yp_VN`G|}Z~ai9gajQEW<-+<+F>szR_2_-=vRfa#1!vkb=^agnt*!JcpPM5 z%`U}U1ov}RUu3`jo0gosx=pflBf%}$M59}#;kawp1=1iA5y=*8AnFn(Xa%Az=9vAS zha%wrn?%nehuWqKrHa)9Jf~U|^g05ZHGg|v*h56vfi;+;J%n(6&7eP|qzPVS@l5t~ zZ7r(5PfjT+=;K_bH`tU?Q3=Jo*u#(pIQUN%sV{thqT{U=$KHGIJ^_NbAfL|OmRQg~ zs)S1x$%~Je%X85?_`CA$ty42}pMO+3OogTjM|0*yYBN2<-O4FkIugzD@X`!>;~@4 z!mn9jD9hdk=hb_-Vsp@iF%hu29c$}BN#%@kziJ}?`hD=OfN6~bK5$pxdTTzH)&KK4 zWnm_xE=Jq?uM1@AyncSv_L>+$2EtHCVc%vj;q9sKjx)YhEvu-xTvhEu8C76D2HY?7m=j$3G zF(O=v5II(mxJ+&nD2x@99zP2;uom-1NM>o!e()my0v*4ae{0*nVUFAww<*}GTO4Z^ zowq@ofhQ-N)81zcgiuME4bBsgpkm!2NTO?PXh>=$Gu9Vm6xBAdxz96NBl3?&x6*2& zH@0h-F;w8b`~9)YS;_T>nbD+>$RW!wa<; z&7)md%kQFMV!9{j9|;Nd+FW_#8$1c*Z(nIx=tvi$CG){5GSR(Rcxobs6>@^#!Jj(p zNm?0F`oGSUMC+`i3mY;of9}kCcusi+0?KFqpE@Q9g;qarGgL)j$AP=rKvxwWp@rhcodCXqW)oh_Xa)*8gi=W8F z6`RfSz8^AlCUrFh&0${tfAG+k>Y;V1S@2bwt<504L^)gEvgTH0ql0QiX|hXgupc<} zY-+7_<|s`gK69iJ#g0rV9HCD($Q;q4m=g0K87kcyUkDwlmN9bd`b!yCt;xNjlAI2G zvi37AB`t=JZ2KVO^$Lr}?*w{$qHe$m9NQ-82-4{YtSXX<`f$>12&B;oe!zNzk>NT) z>Y6zQDsIF|cfBn;hb$qFj4m&~nl~ID=@@g+Y zS(`mYaT9MK!}q#8+z;IU!i2MoqEN1?hVlkf=-OE(KzfWSn#h&;JgsBn477rRn0U&9 z-hcAQ+%J*9a*Di1ynTAa)+5NN>Zi>i`5<=;`_aFdbgLnxpQkFeDAJabh`XKeWu|f#7 z9dwOLR?BtP-vx&I=I5HQcpN7Sx*xY1&_(HB&Sb3a?fVs0?*|tE zB?gZAmsq!!L=*jqF|x~Hx%y+PiA@n7-e%&niKT%X&I%B;Ac^WiJk z57BXrr_VYotW3^>ZG}G_s2b1u5;~Ik6k5$b$}MQJllab=l614juyhnRoJmI+Ba>kg z?l%Q4jq@gF@usx-VdQNRgL>k>u=y5xIK@`ms3Ctbp9Ow_Kyc0{CGJBT?sugK>>D-& z5mXXAchhK{QIAKTF3KD8bdeAg64_(M&?ws+jnacWa0sSF>;1N@1W#gm`+C8Z7DCns zuVtu&@K+`&G^fY#kP!oOPMLi!f%2|{xIcO50{Mqu#wG~9R1Q^H)~hiia0{F9VRzMZ ziW?0#?`pt>7i!5VzQd)3B6=iLrE(U_bkbawX!)lJYM?K+)CHNo8&`$aP? zqY@>qO&tumHXjPvORO2wJ8O~0%5weB^Eo9BJXyLR#8x$r&oA}V7U=1<+w#4a) zduXx=N>)hA<5t}JGD74|<|U7bu<{PwGm`&|(`Uy#cq$>g+fHUNlv!pVTck<(5`WiM zaGBb-jKC+hKp)*V3ya%_im`DP3;D+qbbQh#pwmlh^ayw{8FDucl2NB|FyU5mB;wh@n968xWDWLs%wD=@Js=lX8TgS4AjgJ`Hn@@!~?k;Js?Os zjJ}JA_qJ?(xc+nxxK(0HJ9_|pT{4~@D>xr;9gE#Ro}OMiJF_0P0SzrpQ=m_(*=Q~t zSSJ%c=&9nOrPr^z_DrDZnA@XYu#Oo2ifVdaDff-|;)jP39>atZ9`*i}OjS!G#CJ)E z8PnK47cPmKxOF# zx!qR%K}5KI*oQcC>7a4*`~;z?fO-(JZQ&IRXBjRB{!J3E`7)j2s<4F-T1jGM19TJ% zI@~jC{&Hl`VRh``?9m=;ibmh)sggn0GvC;PSg^c;{0}*Xo2#1Yz$Z1;&?&`$vpYAY zx9XsKQ=%e598I!>;EH6NW9ILTeoDC<@VQ0&Lt>uKw1)+fF++60=>2`ecs@Z2R0b;Y zgB6q6{Q_YEUB+*c z!5xBNpWddWRhY}(kQeRpl~V`2)e1uiHF+{PHs@k(wE27>6%c_O_^rTKsR8YV0uDb| zgEzaK+*#Pu%Iu^rPyx!5&bz*c^u?reVf*Z!m+k|i`Uc~ZD3i6hw-?HwS(i1eUV*$> zA0)-e=5pM%-h;Y!iT+LK@gK}IMjwp*2wZx6{A3ivz+gd=j5kb-*MJJ)`hHxRIdJpz z9Ij`_<9ovc20UvLp~ze$MzYR5`y!Yb_j*MH0Dptv1)a+aApC) zOfYOy2%}%iOlJr=Hyf@2BPv@p%ec;EaOw|!;1OuP@@Smwzh4UvUvAq^|CK(J8i8!p zJ(KSAr?_Bz<)9W9lYbT6>BF zyiT*_UCoI$L`Q!D8G$WY%Ups+)j7|o4XTBK?DVhLK`J7oHcw`E)|-J>WLua5&AQF zWSA0l5P|SJ#Ej*a;Hz#tO)VpY;- zjxu&na57i}P06|~`VV8Ug^?n3K1?mJ1T$r7;!%3@v!FX$i~r#%L`4#khYUk^ZhNpJc)p|13Q{ z#=ha?&e6z|R=9(^!^g`Q#J@)0Rnv~V?EfbH_*RYJwRvKJ5xo;!DuSCMp%Xk-0Wc86 zMblHd6^LYDdSl+1IPo*8v%7mtE~Q81#&odY#S64>SFXi@$n-CHHtZnsPR8?V=MKu; zqt`Gbxyeg$Xo*~(7xqGTZSXv`aytx8W(Ei5%B&k*UW3;TG3>37bo}^Ttgu{d?#ks$ z*|?u7kK;ua>fvk~qWRY3vGXU*V@zAj+#_Hl7{jsCxf}GLEm(GISP3PEn=iQ{#MpVB ze|br(m+keZT4_f;waG2kPKSq^P5gI(HqgE>CI?(6SOGn5dN(*3T!+k$0Ii)0iDM%I zr`E~qIv}1V#Co)w*Ft<30Zeh|=K2Vdd8CKlnHrcI4L!YYnDW6U1$C4zl<9&KRIL?u z6YpL0J*F^iS1`I^Bh=Tu-ii5=HaZ1(-py-?DKOKpyR~kH80Wkh*2SyYW-BOr34Qd_ z#+{697}evhKc_#eBT>0^pDZIwm!M{RZT=dz)`YmWeWaxQEa0^Dkt-oSXSXB4jR|+u z7Vk=5sjewYfXSSuKm%MVs|R)s2Dv~lhr zO^r;MaHoSli)mrBXhdnfn|IVITY&IyK{$}LV^b5+T0L8~iVGWpJJ-z3C2u`HZ+ykG zBVO4s1=5|Ybm;!ng0glTYCzekM+r=)B~}rZWfN60zPb_`E-s%l31eImXKwK}u>E0o zHvm5CQa1NntIRKtnyZA+8y#C46N&-YWF$Qtq$R$_jZbB-=utNhJ>FwzYW@M1bD7?C zegYs*fiO%y#$`LyF^9=%MmXK%2DIsCVrS&*0w#}L zf%Cm{L||YNZj)m@mf`4*v9tNkH|KueGwsHuM}-IQU$Z#CttT-e(t1?)EUD-oXs)x( zQA(|`%S@{no=8NKLVA00lO!rj_?Vzds|+TieuC!pv{tp}gFK5SA&G6Yf0zl!2mXP) zdIm#tM9rW{`c=R~ao&yCkRC%(WNDaItK2Pbq8|&f25WaRr1T?fczNIu;O|#lek_VV z<3`e0`2A(hXMM0I+A}oHY@G>&_{YDu7rPv9(xqFMire#J) z5WRMVn3$-Su*XUvskZvOP^pwR_BlBNZio1`?0lya#;FIru%%bxrwIBF6#V|@ppP*L+GM)1m2qssd*JxOs%z*(iu$yJOwNanw-So|hs54_sR6>qBsZKtl2*B?( z=CuURj2>-$qv0cdBMgdn!efv9!C^k3hkP9`&^jxgw>4C)jH?o)(e~ozlOtgT8m4)G%g2ltbIgnPx5z+-&R5H@pmX|j*60k#Y9l<@bjE( zIr`0>eW{`X1nCF01V=sIJZQ^(=D6b zR=>N&czeWQBL!MHK$9YfyTZ0qOtuND4f%SS=N6BR^?-4|pBoI?#+Xd@-=e0g@4sY>2t!j#kkM2{v+Py_QjM z5#vIb5Q9u+cQg1GierU#KU=Y^-B4*bj$63KnfukWI)btIpZeSQ7=Z82>FV2G1f62L z;h(?>M;tFKr0ks^?lrooI;+!rwKG`MK!HVg4`6DS<^okBfL1@l+3uT*M!$(D1ft>y zQ4-hytp3_Id`QEyt7KylJYx*24}&U0ZwbAuB|xH7YSen4&r>IeuP3x`0k^W^wp9yA zK%QM{CU4V_=8fv-Ix)A^0`QZPK$oL1yZcm~E{ls+`YBH*OaZ~KxEfm;jf8y6WK~%- zQ~R1s(VX-Wt}JFkKDqqqb3PZ$Rp z)jwa;Mh_|jh0CKbHN^R7lO7zj>3)EOM=L%;>olcSa7b&Cg~UU-=~#7He>>vzq$*xn zPpqTw7^A~r4YNc*rB)ZHr3)eFvS(1E$`o6=O>|H~S(m8MeY3lRrkp7q4Yp~1u-ql` zCq$&w{1RX+-L^$eTEsZIDVBgJA9G>JJJHFfdM~)t7s&*<)8AbA(r)Kd^~7a zbPY{=jO{)u&il(WDw7mjcu~CLl~-u%Y>Cu|pRUkEIHR?=%yOiJ(-h#G$)ow%pbdC# zj)x3+1qS8nT#u%Nx zjzeroQ@)8~zr1ePU4fCZY+b2fh9BaT=(FlS$0)K}&&cO4TqW_I=JR+e_PsNl!<60PXTRj5q1|Uex7%!zq!6 zwMIEp+7rr32XZIPtb;~P@{fn{ZXMtt@~fYI)c=x~Q1#O#ir5LNLU;X=~G(*%}0x*=6pfLqQ3dbro=r&r<1RXoh}_up9=`_P!C3uB2I|COF#=PpM#T zX<`^q7dzYdZk-;|D+ud-_+3AH0E#yG%(}U^1xreY@yq?;zsnr3p6v_WnCQU&HvF^= z_?z^LjDT2XW{4wOtRsUsnHfk$mS#Hk5uAkBIuH(NbCLn2j|WCNew2Xd=@92MOZ?z! zM@$~$le;s-W2cT^+bQ(g5}z37?68C7qXVg2No8Mdwx=7KW53wl-xB2;5@aWNJ?|TI z?rlVrlt1d>Om~n$Q85Ob(FHoT?(8B!~U5+lW)G7pEV_A_H*2!N!;sBQJlSD63>EX%o~pzKQMbuC1CdS_d3}koP^!I199lp z$+{g!`;LxZ{+%;iD3!qZO9fLTfI`~F$y+!mf>lLB8@;UwaC54rVqoZ0%V$w*?T*eIQP)KkYE|#4h1RxB;a|C@Y|C!8wT;KA_qNiZ;mZha>|4zLfN&Ndbgo8 zZ^aTWrvRL%;0ggIH#CjY+Am%Lwjz)V%a#0&uKA^`+P08v@_5db64@71(_}hlu@K^8 zqsDX{ERS0}wA^GCP(;qa0wsm9H?5@GXPp}+_+V!ss%SU>mM-R9tzYH%hy6(ISC^LS zE_Mc;^PHUy!1ay?pTCEzSHEE)~pQelQ``Jk1 z6?7*mw`c8QGJRZkcs#g?Q_6*y83_G~(yf6ni=vKA{q!+V$lG~c43zu0(XmIiC7EXu z#`@@V4Xl}MdPFqAqpK)1RGnD3v0wdL(u?gCG;o^TG*(_;D4^q)&J>+MlBHf_ z()@0}x2|CXJ(Fu1H1DY<|7Yt}4J2C(pkBX&mKm)T{+ze$FFq|ff(_|v+O7n6=k`$d zKqe^3SR?up19`VrbMqqGE{>0pur+Qw0>W$pXY!PWWx9|R^Y=Y?-OcNtjxTa&&RP5gM}sAqus+OH z6%6tt-1Tzoo(l}uP#JQ^8zjfv{0PW=6nP7o3QR7AwNB!Rxgm>KtEEGPFKsnpb~Ht1 zSt|!=ng0T1pn6$OidUQYH7l!6rku$4s>2u$)Hf@RosDVFHlgu*HJNN_;JUGd7RB$; zIiHYgP?)O4yeR!o^nql({etm&`&3|MJ^l8;#HMwV3l6d!;!?5zFLy%fOMT$C2BMXr zjqTbC!I=|_BB?;pJ>fm+WLEVCO^gJg=|4hacZ<+CUUQi;oZ{}`<5NBmQrYpqv_wuK zS8^TVshkWi1*enN2X0pjP%~2&MTvFNdseC6vJL+4dB>#?3v{XcyDWCOic=QXuo4Vq z2stx<4X5!|WGAfVWjTG1_f=|#)n#5x=ihAF!?hd*MguEWJ6oDyZaa*-KVh43aaIYL z*}-fA(XvTVrLe>cJ#qik2!$__EB!lF6(AzP{%eRa-smtrElvYYX?{8<4J;Y7&q886 zI=<4v=7FX($Y(a^PmEc!UVSK-8P$Wq{vb`yA!de-BMv&7B}*X>w4D8vt#V0ASYu z{K#T&A`(}puWy^bzM&sx)xH&y1)h=pB~n-LGYNX_W{P1@b7$dT?mNry!hI$vJ;}y1 z;s?&M%HWbU!~MBR=Y?M&R9GmFpmlmQCcWx29{oc7d%J{vdfZr(J#E(sx#)ixr|o^K zQz~-P^Yp~)e8c=zBfDh@{VCUt^4o7T&OXnoV6 z>J$S_&L=PXzh|h^cOt&Iouo~oeQ?rc5Jwt}7yU!(El6Gs3mFyi{q7NR^@mGPaI?@M z9JV|PTc#1#OHHbi?x<|d=G<%Jh!u#h#ki5-RD%~N1ob{m&(0^QeP z*aI)U4mXwarwO9bYW&3Uz8Y+`)XHEWlwtf5AUvUcVbk>0qnF-@jc9jtCn{>`M*aii zi6M2G3uNYw3=ZcWKVUU+Bs3=DbM29gIlDrMr4LP zSNe*xoQzgFX|BRdx5QDRzO1%6sJRC)|Sx!G2F^H~00w+34-`|5?l z1`6xZkQv7tQXu34Qhm7wf%}FWuBQeR84CDD3h5K7fM3hCH!nEL5lm1e|9Lc;RJ@GOaP$dc&ivLqa$H~E>c@+mK^Vm@S~a09KPZHkahT;+kc--#bo3fHC|r%wW}~dcWD(7gJF10 zwXTx<;l7%_tLL(*8?@3)IY!uLXA8ERK3`Mg-?@%h5{RgXdseUVr8KK8v>2<0CeID+ zbB5e}yaqUIzR##Yvm6KW2DRa^JOynKvlxNQE6RLCwSEh?=x}v2vXL)M z&2rLU6y(r8^-BTD!m^w_yO+6DqSb>hIaB}rRL<(f!uL-4L*0pSD(!VOU>r@9xK>{$ zfT|vz6~+Mf8n4@qq+@+0VI!ECHInjeDBIZ9e{w2}Se5&T**hmrsFN0` ze}~iBmp1Q{GP}l|Wf=c1+cs_R7fi=(NSD!2c`nn~5s;zVz+p(7q=)S60b{+%9Tf_U zrW7AMu3hlZ_|enq=M^~=?QXbZV=hAsoKb*QKM!w*tPM|f2VA|oAy3-G<8;x`;^aiF zerW?i=Au7jyjKp$RInw>RR6*xwYRavt)r+xuX0N+l2la5x#!Dd)|pp)`mj1LB^qHG z8i9ANSllfFUT<3aI~<&s20U|i&Jg}~o7X!M8n#f?C^>sTUu>a+uk$)0;I*A+ZKV^F zI%r1@plJpvT&^r1gi6PLJHqv>j?W$_DMIsRHuC@oK9f&X*X6J_@CyXdR_Zdy&}T?d zF_oT0bOL>O`>}Q6+)xZX`)#l*0>`MMGt@$Lp8~LyVOT0OedGy|>sW`evx0mWW{R-+ zSz6@{*ew&)32~j3Xo{~=>v)5zy60B8`FTF0uI?|3Q^uXg8&hF<4;Bb^llz0Yt8#0{ z>EDlj+uwJ=RTf2A0&(<~lV+}EoGp`P%$A&c&(bQ(lHgdXaRxJIm<*o^3OJoT=uwQZ zOO1$1>)M~76%Gu$+J@hxHqUA5d43gl2wMA%OXX|v!GF~7kP6_~bvYUSCgzmRhu4Jd zIbHe@rZ8@$fpZI_@nb-N4H$G|S&>Uj{Ay^T&xuN1eLEtAcWlQ!+>j|fJ^2WhDl#jk@2cn ztkEbMCRfnIg=grpukQ;wrlT)6JJsP4a3%nF=41yxa;| zRQAqMgXUqKsfU=fjEDRHfIZX`=(ATAT!1+07~-_+*GyJRS?c4lg>2$SR6@)=zniOu zb|c3q zG;x0f*o}M$G=nJQpC^4aTEo&f3zrImZTPcI=0P>9eh{b1QXbxq?TmsQT3)DlQ~IsUAMan22~g z$8=UIEb?lX{IGU#9GKSZH?&SU0pqkC^VVHgYUA^h?N0YFsZBXt4%CxfZsAR_vXRWL z-5_+9Wb57hXkFl#1Gkb{;Gc|F=<~cT1Ahu11qia)CKHuYU5&8G2S<1Y4D({#V;sBleGalk86qruav**XoBnNNkm-Jw`VK4SjSO{ zT%97d%y%u5-hEno^9XE)PFgjv4JcTe>~ck$`*y_*AB)4n?m zW{1#INncKCmrG7i3=`JS{;lSLi(F&sjFphsmEsV(zy(1_ti}(CEet1~K%a`j4KT$d zYfC=P2ABzhZ?#4*6!Qrq>5vQDeFVCO8rMGtC3~UI=e2y_IWq`ZA^#5mfVA` zGuIohpzx0BZ zcs-FI;x>gh>~J9B7)!n0j1%4n{st`7wErIzzUEmG&s03DYqIYeu0g46e#NW>*f&xa zM}6%?2_Y)g8eWaH0-GXBZ5r#MV9j@|0e)_fed8bw-|XHYh`NA&M32^F+H?1?Y3E>o z?m+gYnc3QjOSi9mX2lQG7Jg0kok9uF1eC^*!O5B>5QOG^*GLEE#l@*R@FNW_u4oz_ zh-E@EKIKsvo7fdkbwd)kEfp|@LPGnPCn9{HBU501E?FwUhQbuEBY*Uegn~)+glq|d zdZ{vkYPIT{z4p9*zfW|+a(mw3E~)rG&x0s3Lp7iH4>9m^DlVDlO(v5ST}GZ$F=Xr! zUBXEHi;z$(DUZiQTUB^G_Zd#+2Vx0LQX#VQJdMVr_w`nydE;$49uB|W!tUGB=tq0d z1130EZ|7X*({lyr7~*<7y#O`0zH<5pLX;#f5vT#_HN&HZ$I*JljFZWvyh~z6YPHQ^ zTinWGo>vsPaUAnZ-7s`WEteuwLzK%s)mrIn6u##fVAO4GDz&Bt3q+bSnF#c^EzdSC zj<42R?b3kgk|S6O3ib`4I)?ojUea_J@u}ISq1#SZv>t1#pKNTDG8(>(qtzJe8#_}L zE;DOqtu$3?~@qOR&=%p>?n#9_1=eTf{tk6H zi=Xt^Y83Z{468u+YHOfz;UY7T88vuZfp@}YV`7y4h7bi}>E=h;)2-(*%Z|b^F@GEB zz4)d`aWHnw{_tZYE_eaj;P;#d7Xu9c9#_U6?d^`9Z8C@c#vqI5r{~8fZ(kpty*zw( zY7Yqf7;E5^dcAu4x1(zNSK@+ezyli9PZlVy*|?Nm`E#Bv@@Xzv8i-iEeb`3z_pmT? z4wqDBEOmFca$K5Rk{|a^51%|LJ^ANu)-28C>Z-3ICbsq=0@t>CRY;WOMcQ`huID%q1qgvEgfS%`Oai}`b({KV3fXWQOBiP;rNXiAqXV<}ls zNv4d&WiQb@jR0!|R4P8PXUf@77eey`}luo|De$QznZD2F{ZY5VFS`kzr@z=gKEj+s+E-Ag5megLAMsxU9)23a_`{FtKk&ES_8&BhTpmJQ4$rnFxkSfd1>J+; z1MN7>Q{x{DrH5fpR^;`&(=&A`Q^^>|L9eyL3bg~PT+(H>N!1=>W^&v(XZ?n@OTL{Tqh!pd2mGUfOv4;c^b5(+D z6a-AlQaq4P4lpT)d3=?jv@pU#7yy;pm;~B$v!JIsQ8Wv6(wortynT|gKrELm4pDyQ z(y%)sc4i_8b53~&(khUYg?V74Zd>1^ltWYPl~*1~omrly=#{C^Op=O(rO}Ekd78pH ziu^l{s8S@$!?7G5j3)eNakdw6gp;ul4x38cIx zmjgKW6mY05hO(tJQOG6bG%N0`wXS4>ENKi5Zt8+LsHTYhp<$>*7a_BGsxD=bI1Mo1 zMK$DT+>MhE$OeQ|V4?y&lyOl#wdI&RP>kXO(zhZ$(*K!A$E5X@d05%?xt>3n55lRh zK-Rd65Epcm^qfJeQpf8RN#-P+I&G{hk*yLow!3G)2H9b`8kiq`j|Sm%{P@wsy`qU^ z>6nDmLCR<-;%EiwMC{MtEyVtZ6tP)``Is9qCJ#>EB`iHXR6Bl5zxqmj`@pvRt)Z`4 z!IVm-fWv)u#_{6@%Yc7Ha`p@6Z~orQ1h=@9!1U|OZwq$!r}uF$8t<| zN#7!f%g6qs|5k2!6h=(9-0Nr=h&X;tKfK~IrVz}Sj2=DO?HbzbdvVCS(Ei!(!|$K@ z4|mA3-ACU)^B<{yJpTTf|5*Lw$@kCvCp*ye`_Zm5nwnDvG-BbjP{!e>PrjQ95&fW5 z0LEEt3LS--{+1oX@x(O92SZ` zJ~`1XJoiEVl+t9uGuAbrvs>zoRz@-pcfP?$NO6gJ8NTA=KCF~LiIO(iIz^FOq~ym` zT*)H8{=1{?+N3zt7)L4Q74M;*Xid9g{Da7MqY|g3*wrhJvV{ z4~k!Fqb+HAp_xv&C+?GZO+SESlfbo{>#kfrcNMoV){-VappTVAosvmd6NvbdL5-bE zp!yY*0GSb8)K@@09EER%)R>oL(LrzIk9x)LStQ|njPZ+(uEeNSdj7zVG)+KY*>h^T)rjRr$BeGGlQ!8qr80Dgu{H z&(|exHRPB57ZbLsnQFhNsO6@k1PLf+QK!rWz#6DbP``1?LN%l}S+Hnnf=wr=l%rWG zE2OL*rJ4tllxD^F_aW0jQM{IAR%9Gc%Q&QvaUfaYXAyL2DYv_rr<#dG1chxDLwvfK zDcF^PyDI~;D=sHxPs_EUHuV}H4nbx`Ml`s9zjGdr?U7@XK4CmtXaPc`^K)+PIDI0D zueC^{M9;VB9MKGy=fc#G_#i$^&3AdCqAIA>vE_3V!>IBv+CC-Mhxk}GS8Tjya69jNNI4T7!`}M$#W6pOBSOu z!3GFCW*@SmxUOW`AWsVXJutrv>?VS(Xi&oNupNTTSf*pnan+i%prfMUK%EkiJD82! zws*$_YvI=2Ru!^K7KwzVa;Q3-gML%5=e)dtZQuL5$l(~l1WFoq9d2WLj|hgQ~==J_wN=()q3QpGV^B$POVp^Ie(Z5k`mLNQ!gRL99h}ZC zA7m=1mU^#I=60PQ&Nyf}MvK8+&&C#gZ^P;8!8bi*Y!0~+3*~L?oPqd`o;=xm^z_@^ z!IQ^N9t|En4uiqBdyj*`x6_b?k7)R8FML=KRP_o2;AJa#u1r~v(DNY#KW%f=I+J|^ zZr-{%ssxnSj4iiG%I0}Q(`W@ypy!CeHxo1e#Jpnd=vqmxYS6p>6&zT7#QcEI<|&;P zw+sB*G&`-R86=b7kjl|)7|qBu=TV3zA>%U)5{yEtzU$zo-i+hkCOfyieEkap)crfY znJLws${1aa@G=hI;0mRu_S3SI1)2_vM!NUj=Dqjoy&rDg`(e3viJ3I_seJ_Pa@pRi zI;6dbHyg9u%_HU=;4^T>sy}z#@)%lnJa|M@r;2Mg!m0f&WdY-tnsYCR1KBGoPyPVLpBG3!5&4xFGLNUg>0eGp>_GRobGHcduati&dBmXu?P+atLY3)vKST z7%QRQSe8U&md7Ag`IWIU_CZNQA3u`G$B!0=ke*($uVuF$@yVx86Y}ZPgX#c0Q%BoA z0y<$j{gHSs@Rehw7#{JAYl!i&*U@B|N1~}>HohT8aa82K2Nx~vvE%TFIwrnwU0bgnvM(r)Io(fe1FA6I& zebgUdAu@`w9P0h=pl>!5G7>Q>#ReeN5|=DhA{f#_Xbs06qtl|85w0)PPUDjT{TfZx z1or5EDsmV*bV#2IP{S0R3%RssLdsgh9LJs~3bKbTe6 z7$UKaeeRzqdW_C?+BeG3j!1N8N|{bAJcj49CNv69i)h`(b+rvu>AM zIy0fTFylQ5bj&YUijMdj$1xshNI64`?&(7!i81NLa~^*{QUU)oF>HRhTgu@64=_2v zm<+!ft?iRbHb~i=eHfFT02(D@6;Vyib)Jl( zypQ_l1s*-<9&Ag?sN0JWlZ;l`6O^jKJTHwi zw0%Am18C@PJP5YZ+v9s0fC8OwS7+W*v#CtsUS*8ZUh#Ds4}!4NbW1N z&V@Cq)CM7%x!BLLluz@FIYJPQX13@+6R+jUuG~mFu?X35K8?6s;G$31Y+MoL=*8tA zNSO)$1J+rcv`J7vj-z}&IEwWNS6EGEYWuxU*-~Vzl(2=z^TzY*&ObPuMq+9r3r-IA zpT9ozmtjB6;{lHcm`%|+)KWHuIuBrPd1Y0lWCfo0lc z_XGS9qz!5=r7vm931XN&1o9=HFUbBSG>jC$CL_?)rko%X^P`UK#+{w0nSd*sIBeO((~78lNtE1! zYFjm_ss1V%R5(;9*CC26qv2|<=*k1B!BhxcQuOIHtaJ_fOb{^zUtk(V(B$Y?>eild zTPA-OxykCRgKx-03E+_K?$f8q1PNN^RN(^ z{_Rir!SJJWP^iI&fR|)LRC9BH=lVHzcXxN6JbDEG-QC@-{&#nGckkaukDfezy!&|c z@bTk++ueJ#xBKwl$nNLZfLvxY{kPpa&sAOAKgs9gr!Nh`U!O1eX@CD04<&jJ4-eEA zKt*mh$GZ3b^zo?j{&z=Dp8oazKgXw{)(L1vBQf^_nq?95xfm{Z7_!)^TAG?%gPaFR7LT}M1R}d&avm> z*v@?($d%04vbkf5MMDP2z?;XAI}qgtMaAaS%{ zd6xyGd;V@4^RdbMci6~V-LdZXKYBcR^sxH=AMHK)>-~R@PoF$@L)TD|U3~|>=jIXl z`~K@!12iaPIzfYK@SFt^O<7xQG9DY%b75NbykdHkcAffPQ=tWh;>DmB9^{w1qLSD& z;Dt60f<9*v$ndp6p8_H1uA z2xgR0`f!(i`#5;=bjH3N4U4(@uSSDcqsMAt2d>J?^X8G5(n#ajEYd+EiA;{D#QxGap{5WRBJwz6DGSL~n4q=AiJ)u5>Pg2U&BkQ3yZ3lc|DegMX-D+sw|~~&Q}F|j(TH<; z!4)gOXRCUfT(WXab2n7VywF-bGYTT`3sdsIaD{LQ&|Ic5d(K!)Go&W2Vx$uJ41RXwak`+oEJ=Fk33h zE0+jrvNyUY%yg^7ftAwzRDhaNiD`+{Z-=Hrl}uOwW1s2(?ry?h%DFXu=jv8VUa>=( zaDN&0?PAr0g;sP23EI0f(t#x@T}uBPD}&j!IT*z*1yjA@C5Ii;yMmhc1WrrucH@EJ z?#)v=m8KXK+FhkNkFkd-!YEIYGAI=0xM3RF`jkDFu}%>r;E;mc}!;};}Y@j4ooI_95BKGTJ|Af0YnI*MZc(v zXOInE2z_}hcaA}GQgh~Nz9D&TnRy_)xG!V!@ssCSOwx3J?dvN_fZ2Hr7sH|8qRfh4 z?AsI`(P#SX;ge@D;?a!cF+@IO*7MPc#f8|slULS<0~sfCw8oHan7YBoj}}{e`ZR#X z{>P6BF(r})?9-=uW{UC0kJwHD_URKoC5yb5B1G-Fq2YQn6OnFRHzoi4?N7DsBPdgkkyp4=y<+ zFVD`7Pd!Vg(ChBG&{SEk&?I1nMi4ECk>L(t|GkM>>>kZ7QIQ9RFGkHkM2a)vm{1@e zz{Dzs<-ymTi}f8hZY37nW7rE_VS=@#xWpcm%<@bT8O@ZN zkaI>;mYyllUhs$&rqYJn*U$l!emAs!w$Y8_`?8JKBYV$3+~#m)xII zow%lRhic1n0JX3h(?D*YYMgKCq$(FMFs zFn*w}chLbo+5<%cl5izdLs+39;&`Pnh{ps-LN0L!^Fjek8oCa?8tv+JTC}Rsu{sam zGnW`Mj=(VhJq5m!&&Z^_X_Z({Xg_J(S7>79G?BLMejux6k|89W(!fmac*j-tRJI>6b3biGeo8eqR8LQBz9t_wfFd zDzm6vHLl*~nES+e2fAbfjUw>i!Tg=Tgql@5I{4T_Kc7>V$0x+8cLIPAVhW9zDM5Fs z{YGu7U29G#IBw^_GCCiV%UyruKOE?Gd&Td8ST1QCj>-F8rE?E9A`r<+;Q=h}-afNQ z=s1Y@G^OckFwNy^3N{7}0+?q;o)uVZU@?R(=+u%`eX!;1Fc}xx87V3LjF=)7*J%$7yinXpj|HT>)WTqv>u@S01(XDl$V2O9M`k=_ zR|>EsWtTAcu?PoeVgR-Kv`$zvYSSEoA54p55#mB#n#g4>@qi0_v{L+g1+a=HM{8Oz z-E2(f$b?cy)*DIg2!Xr(6T*VudcXcD-JbsX z=+7qhUu&=%o-bIGuvGe4asyNF8vE~~Cwq@8_TRn7fBAoWo==~gXvYfgP=JHVwlT9Ew z7?a-N?>UWn<=tul!)UfkXN=ZHsE3s@()(`Yj~)$nd&EDW2{nPT{@)5~o&OWhj9E{O zGi4dA#BUWoKLA!F8MN$d)O_kcp_Q_{B#8F^TZ8?uSlCzAY)!`BlaL3Q{R!-y6{j2tUb2klOJXdosSvm(3A%=-MfEVLy$BnKR(mJ4Cn>%oApWV&1 z9wAoMgu6;e1u4|*o4Oe`T-T~}F>I$3bk_21QYE)_(+#OYehjt?#jA)I3jOIjp}oMs zTf8(2`Si&(ud1Joyl+jk*W~wHWYE;e7GozkaubM*A){A0tSyt~mGsU`B)k4fn$$Q3 zpu#Srgru@s!cL&01_}!#1f!#12|2H7>I~DcKF#)a6EUcD#5NA%@3prz=Vwmd>8FiX zUrYaM>C4Mt{B_tPTU3)0l`?tOEwMU?2Ndp54=rVyW`b-vyW4jbj6&$3$x(oHt2^la|5S7k?x7V=T{|7h8~ zujK&HMF}H+j_s>J=k8xiAboK-fW%(mTKhiHA`Y&t4RPZ-=W z@X8#2kA6?4MdAG9ETiobkUB-imu-% t^@z$d-pIutj@xr=4axM`}d?t8AUw8VB4 z>YcHC`o{a_f>jnU?bvky>+|*~Sb}?`r{fxX8}&l9o_X3^09fzbL=T>y_qI102Hmq@ zm>%ir-_Lot84w!cC9oJ9Z$^D*zb7A_|r+~DL&8*{s}bEmGKp04b$1(jh9Jm3W$=EKzg5+*(liH?{(F1m2fTC%ZM*2+OqSotr83)xe3jOHxZC%EJ-J z7F_tYVYVB#zPv3C*jXgexu^?rw^6T7J9~}9cim!&LCtzL9Jy(I?yD~o<|e-)g;zKNx#UAM8?u$Qfp8y-s;EY<%8 z(>ZQT_Ua-^Zr#V;1zX((LT5_kS)7JNa&P2KPejBT5}R$T^0PU;RhM>QJ+exe zYZCKVy>9YI^-nQtcc;lSk%_y|0ig;12@|`M_YW|!(Su*h?jGv<(73vq1jf+llBH8Q z24~v$4|2&$J;C&W@jf@udRBP>8i#8%JhxDnz{+D08!wl4C$H2xdqeFD)^is3zBgxC z@7Fft0bY-?51`SbA@1C~M-vjRCepjvb^MV-LeejKD@DX~0y1;gI#%HkQh!r7hFRti z*rjiuNj&J@2F%=f)14?~DyOUGT+_J{lc7;vEi&QbX5Ft1;Vbt1N#!~cNF^D8(vU_!IfZ5PzP!$5 z277}^^D%=!QJ-MR<1s1f3NPu0F?sU%@x#X+Av1|I=AMQ#Njgu2fKV26`6|vG^L5(1 z%?~ib>E6y$pT&SY0G;M@o<1;Nz!jv~;sNCIi^LTRU+S++m^IB7$B=^?POq>=zF{o1 zF_*$5G`I0cCJn^t=_?FG!q^!dr1r}Je4|9?06Z?l|0>+=8X zj&`g0{~kSh{FnUqIX+)T{-a4E6>r_r%Sj0841h$^iN<#|UiwE9+2B5fdnBR_R{#5F zUBrhhY05dt$ELgm{?NXoMkY+#i zkj_D=&#gD|TT5>BW*Ha4h7B{ZT&7DYx~o;*w7nW8t!x`1wU`WNJ&g{ywY9Xs?he~6 zAD3EQlj_(dU`x>=EwGk+mNy00b))Wt)@`^3*e1qOeZp8G4{fF17d(~OFFaek#4fsF zS%7(SbLq8qvoxCs5jMwp6rI}K^hKA-j}G5tG1N_lt8I&o->r?@6pUygsw}-V3cv@9 zUpqG#!!P2A$fr-^)@B)<7w{GlMD~lq=cL+SH+6e!g_!Sp0*&+#0-fY8rQ9P`M10MDhJCGF{vn z#ELv&p^5OP@38`gjDX$8+wc6-Pav6Ux!(m_t?4$nV9i|vy1}G_fDTl%(g-c7ZaxD!@D*c2+L0yZX1h!nj2 zyH{@qHF&E&a+gOiI*Gai8k@y|h@o1aGajgG-iTO<^(A6#e*|3)8SU=wR`=a&u{+uTgV?K1b(0 z(LvpK?RvFQ0w^E+)yIRwle6;|N3Rag-|W9W{8VoUL4RL;{M+H*JDVZCgc0{^V>+#i z-%j7YJ2^P~{;RDSk3)U60p=kYfEUe-Kmu&it3yeyiBwo-pu~m+qahi@WVG$fW8uBY zG?hiJpT%A4jn-4!qqe!6$ucH0GWbX7&~VaEvX!3z)r5fJF zkd^&vAMt{%HX5+LH62jWMmTGi>_8-t>A|+fTrgi%XHo+@U!Lus{B(Hc-q|2&=v3=L zuCMDW3>Hffl1Gmob+&H>-E5zRyt3^8xc#mf?rN86`*l#aH*eo>THEp3>EfR6uif=} z$A(^OcYc!|jVxCA-8HfuRpoh+r3*Jeq$r%Y`4ElM6l6$UuQz|825R7kN(g9k;EGFi zN7!^;uO`r{C~I@*4)*VJ_iU!IMj$JIQ3`$O6H;HkF;XOk93JxudtQTmO&=Gv#d z2dBCA!OK2##9O1{>ag`TTYB%=1x{apwGMC%?LN2r8-lRb`K{h;4WKvhTx;Y)|5+W^ z%GUV1SeRck@_lyi+&#FR1q0z`J0b@bOQDv@N48QX|*t+m$%? z3RAbYyVyCoHcK9MI>TQ>Gea*@k%>S=V{&$Yu0ohA=-5oy)>SrQ&W)GduJXMl|EF<} z?zM&yT*v>7o>t>Op6ou^{fqzm9G|GE58q)G&zOXg z8=#vt=@|b2w9}F?vK9XnwL8hjkL$;Ck~}u>p}&bKu$1GXQVO8B@T^Egrjec7U{GgY z-K{yTGp<$v*{FhS*|>WT8!@V*uWXId@%Nhc^((&OE&E z3qYOEFjW#dlAPN8*>~fGpD+Ho8UF`(k!Eb5i@OhkG+eI*T*Lp36th>g|L*;j z|LL=Q`kF}1GqM%2m~2HtT;vJqb(sfyr03YxdWwAXdfVG%7kI=zNRMAZ9nDgG4g3L; zp5l+aHqLKY=J|T4!gmJILUyT|#{Th;spWvRRK=f@q8VZkxc;N{|%P!j{WCIC+jG5}|lwJom{l zA(G>wDD7KQlomQo*-WIiUAfAd4@b9lZC@xg-Cv*2@VO=aPh?_fz@72`?vvd* z{(txn*w(7lC^kt&z*wlTbFYjzJO2Hp&wrEm?=M)i z{fHEd;gCgR`Y-EK79Jcniz1@lH|lkGJl`QHi)h9#88}Hh-_tns`Xpv^EUCMdKoTVuqHpy7z3pRN zX%RzD1rkXZ6y{^T=RZF^Kg~qSygoUAvMJ=}gHsanRC@lLXG8dZ1mE+g|44`M|K^Lu ze5n4<{3$QvVKLB@1`6YWL$>#gFRv2s8-GeKyl?z$ss1feKKH))f4x5WnWkLik{mrh zl%AiY;x`s#p3g%@huAz7zj^+p3`EF=U;J@z@Y&@3pB(N#e|_jL!&}bL{r>l!j2`Zl z-~Z_82tmbdNes78)|GEVs3gh8$&a*{61@X<)5H^jr zvG%oOnIw5qu1gK|sBLLwbS0s2=oYC~ZED4rMNDX^L^zUn-}8=;E0yGZbzz;r>Pw%c zj1A|DL81`e&|tn{b*ODn*QJbRtPUHjHlVSM23atkf`i@zNn%|RTrCpq(omN)Th#W! zC;Q}mgIL$N43wUSiETbGML@SJ_7ie&LbxO}lEUaci{j)~1L-CPQB0CiK0-_)qrpO* zN>OiI+G5$Z0=DmY@06rUXpF*V95C!;`6eJGJfo%%xavC-Q6#RgL^_KVx|L&ZK>jXr z60^p9B>)mNCA-x{TR>G-#pF$yJ_IN>F7izBkQvY+FlUSTbH9j_o3JbhCw6SEuGxa7 zA&XgPF4=ZBJa9E(Y=A}y?Ftn3ArbJ53Lr*%PyJng*B{lOxX7mr?a>yNHM5%i$6_bB zG{L(PKw+UGl`P1Bz>+3@rAqpmm`1g@qhEY(~PE> z`kBYIpLfXnV-XS~(vV{vUQhs4&MXs(Kn?>D2Q10t(D-`|6A=!~9|J9TZKKdMWh^Es zTZ&5-;uTR_;=r+geB^uHz9dwMQF)Z@kZ-=xLQ>wfv~RwFBBZeGvy{n&Rd4YMzQ>)g zFLHS0WO!>5md-@FL>x4Lno1giDlP&PJZTkGC!iiwHx$%$6D6{8B}7Wx zjwEH5iZhH>wqcg$Y~p+W^`HNr_XGLnn>Qk3<8Qw4ejrCB;ZfI~Rs8YF4uk1s@R40n zNvM<};BYgMR>UhMh!(y+4JX56GNYGjI~9#p2q9tw7l^};yVmP_Rj6lVC33raE0L#G zsBXHYMk)qy)Omy8jJRQ<1WGg1glw@(7ONdq6D(<*(J0$09g-l-fruqkBVRC!ZPc^G=+P^{Ze}xJ z+|r75!YD;=6A`QH2dJUzE@uL@&O5j(`T<0dzB>)q3o}F_O5~Zk^a~oziq~)WFBA|t zFg@0OXh1?5Z& zz{#mG93!_G1(iKHxykm6_<0)Zgh`Vn2+(b7uOX;A9j; zdk>#Jo8XdaHO~!D_6N|!Xu?-K&}%a3^(K0&-|DJEom>4}AK7MG-R!G1fpOI1S8w$m zdGq=-g1G|6k#f4 z7qYL!qtLEfgMh>6noXG?>VOgv%HmC1=0kgMKpGt@4#bzX`m#z%;a+hjsx| zC$f+4TFeSZGR-U~EUu)UFvLxPoCDRY{}H`o z^j3WOwY+TWS!7;^^)4}ouMjuzcp}Nc*~eMx14$0;#t4Yw^bvg&w2q~zVvce^1SyL{ z1{Pm!>;mq3MkXz;RTB_iSw?6i;yDO(K-qw?+ELtAz!ZbXrNfWn9fXc}oir7~{nv+O zl>i6&rlKN5%{6p2C)Z^1@sr-O8gTBLEh$n;Itt*Hl`J371jvGz6*6dXv7Cpb;}G@A z-1k$yyKaI)1xs#8<2pP76psk20=^Nud_f zyC~JWi$!Qy9R0ha8UTh_A&`g%aN4NSz9y4r51%}%u0!WSRBN$5u_YeIw6M&Mm{|&8 z-7MX#ES8Ra`lO@jAR54}b&kZ~(%>e=noUX)f&*7Y!OJt}G#O*sqgi<2j6)~o>1=+v1pm-w!$6GE=qVqc8b$LNe$ zXr|YCAZajRS)LVr`Vj-br=9-0Q^|}po4Mk=rTAtE+HhE9y z#-+N#3lVB0p0WUyUMq8j4UcDNJIDb`x>Br#qhFw8mS;H#`6%w({fwG+0XP5@E2&AW z1P5S9QzF9qy4QdpZ;_0-ic@b>`1d|8=D)_RXe>bd<+kU}E67#}E5L6C8#qHqUn*u5 zjXFoAi|%3ZBq${`EmGGOFV+oiVrQw`jo`gtD`L{%!d)&j=&PHNG_{ypt@@_VZ+$7> z@KEvrc)%GDXuHsi74*^9=YBI9}9r+fGr|1c1ykxmm+f)+ytFyU5@e!Ib3}(0{llQ+?Hm_bc%+(?} z-~;Y<^iW%Kv;jC*O0~}8kfjl5XeBP& zl!56J^s@>g2j&HpJA~Vt2R@2OVBBsstyg)34IWi}S6u|N+rAuAUd$}VmK%eNgj#zS zAr`a}kiMz02*C`BB(!}E4Sa7)TR)Kb>Qb26^LaDZ-BZWG#d>U%+_5>);;8Z>QZ zbv?hzy3<0RSF%Q{ou{qA3it0p}H9xQrpO8Zv4VIVJU`$!^IFv&PgKWcQYa8r}E+ZQyE`9Psq<9&$JkkW&PTA=t zijemQs>GPV-a($G>N?duEFVa6i9;gVWD>juv=v!aS6Xlgif~-8l{F_UX`<+yniaeR zak!n5QXSR_Hwu>JE*l+W^x{JnTgz9TgoU4sF|$KYFdkTwfrUP0gvCHC8ARcSIWPoj z=!hnu@YT!0T!h;@8h-3=EpAB?aa4eaG-Ro-FNx=UbHbNDYh2zW}R{UPO(o8IyhVJ;Ud&uz{l%))%HoCbJMrHg2M8jlQtD`?={p z^e))yN~9q;fY%1MUwH!u2yZ}ai=`q!CYRiRuqEZum>?Mc&uX zD-XakY1O>J9oCes5;jI+fm%uLve@s3{^)a~&CTDxLw#>J$GZ5RC%d)x|Mbbjzuy1n z`1Gxp7@lwCQ4gx`&ISR3G4AjCuU`#jB3;r<@mz|(Q4OB6AX0qQB~3Zn@asx3aF(=@ z<-X@(B1cH$jOiN3Box5~OEH~$k)LsFoE3*lo~9yY;h+OxkC1-FX9H#;8J&}>g^&fi z$tC&tk(ew@mtV^s@saB7TePTCu+sUM^bkSzJVKH@icGobquCpg9j8oUmaUM_naswd z*Yi9y6OYODz#}9O5z0O~qm~vU$K=6#4UqhGCkv9DC0mMgwPQu82YRpEES%*1uU_Bg zAnL*I0|nIp4H7-TX=WP$npqx2wJ$yN8Xm*0oe8pFQ38fAnsErs#>$68WHvlZYKbkE z#=X3n@>BXS+&&0ej6+u%6Pl(JB!`bC!Kdl)Nl6EiY+ZaS2QphlY(_Ep-Q+(p^n+?$UlK!>IXb)cVUF{36mD4c|0ydC0~CB?O(f@qS=ZQ~g>`>>E*s zrY)YT9nu#aoe$+`(arAJ(`QdS%)$#yB(*Y)}Sr z87O&PT}arqLU}9Q%sH2P{=>)qF1p`O#U(4jPGR)dm3-T3GE-szD;6-gISM=JS4{Lx zIcfNX#~Dj6DQ*a!T1pdk#+OXw*_e!WrE>xrP~YTTEx32GU;NXk;40gBqSba=hPi+*$SMK2HLU#ia{b`N1<-?jZ$k5%I?ZRz>z8fT6>$-8E=utt4eLmZqq!lw!$TiY^Nu?UX5= z_>!e75~*jQZ#=R6 zhkQ0;pb^LGiSWNrKAur57Z64?x=D)IlDeF0FH@?mNyScW;D3at&Yo#63J2;%Na|h zK{VQO;aED3MOXw9AUW_`n66*-EDjxZiEHaiZ6d8WlI4ZeJ{2)9LjN1S#();-w=+cg zSpFDxXgwc<)2ru==dP=qg6~R}a;5`!e-i+sKwQ65BueTL1jS3do4E<%prIB7 zw6A`tapv_oYT&EQ=sQgUc(c0*X9UlFSs;(dXNu{dBu z^g+p3=rkM9gsZo<_h{Hhh%ea6yq3Y`XrP{CRSLrWaW)J*&&qTHFBRm-Dt2kX?5X<= zQj~s>CI42bFuP^fxvUKEM@p2fi0PEhSP4pe@$NUydfuw0dsg0fck=2E?AVuBE-nD( z7iy7vjyOpm(X|qG*_@b2F7=g!tXSi{(X+=sc@7az(ndfTF2ec}G@NWn2*$aNv=~UJ zaeydzsY15a(hL}~6^~Jbbrt*kU;p{PO_m3Z$c%_-f+qcD!Lv`a&`IblvkVHnYCR6D zqiOn`>LcC+fQq1Np?E1`k*cp{MwbZ$`DUqzfGq{pp)*g)6bzZP+@9`}p!yZqTP*=9 zT7?rY$`d$xBt5xi6@rDZ5xa==|LZ^hcfmKrpuy!*=ZjFZ15-^FETW#vunW#S)(uja zR5dTp&W`si5kx}jJG|Y?EKByBZF-)H#5lJ(X@=4~jv+tr)(grbbu_1W5HJ?9aN9hz z!?@05HxQn#3=3Bxy@&)2gO zhR})TE{?hy8HVW%V6Vu-4{GgwE;i808^r0T z7K8*1iebCO?Kd%nTdIh#9+*o)FDZ}UnwdEF^Yd((oENzC{5LT@2WHI57@PAW#aBbH=42JV zq{5}>m7e3*txB^#dF`srHrRO{&zUUPsb;O4;e`H34OT^UrP1pjq`0bRtTgeuCw-N9 zFt?=YWp!0MXt!2Y-3NRPx~dyMy0NC}K0s|$RQ3DhAeWg~@_(RUg+6ennPQ3OE5acp z$SZ$a7>iB>ig9VKPCXHn> z&0~nUE@`(Esg8J-K}GO>0m=^Mg`$$Y<}p{)&o^1GLQ&*^(JOJfYw6|T^5m^uPQN3(OOgW4#L!zHnjpA&;`EZf$-zNh=nLAvdt&@sI*xs8nvj6Vv<=c~^v%mkZ|Apq>p68`g3YCWX zuq(KJy&wM$lIY2g`v=8!^t5KXwEJEkl(dxyeXZgG$8VjN)GJjRH+@eRR446#wh1nU z96c{&ea{vk*PZ%gUrIiY2_^GXTU-enj)I)+p_pFX0=##5YgcSGpPC6;ao=7dLZ=VD*%$l+ zpi5BROT{$*^tVHs08i(wQxj14j|V3#~lavBsB!yA>Q)G?E9 z1gN9Nz2Q;|9|HTY{hF6T-wl4XwtGjLjfzDn3e}BuU3ZP_NF^E-%aFxQV&Wj|Iu##0 zJ-${+YV=^}GI{={L5t+Bw!?u5w=Rrbby6Q69w;L8%^HxAzvn;nA9-pIXc{b-{Uw$o zH;2q16idouUwd-vM$45S#?r^WzRYJc{=w@LVGx4zM2r1u6A+6gAQnm8WY7JvTm*JB~p66C{hDvP= zY%OE*MraIz2)TI0hftcQ4E=G+1>pLKN(ohrA^B7&4S&@qNwods> z0R1ubn)cJV@#Ru*4hGgq%&E8s|CIx<(&>ghfalfG-ww$G`Fr&BgW9Y>%48+lr+8+? zZL{E6pk4}@{R~mnuVD}apgHO9mZtvn4|pX^Z1|XbXa1>wftLCw`)~bQ%4n!~W27M0 z<_T5zZpP=}9EJ}_N0j>t*y6{EJKT5p%WIkxK(;_48~y3NleTnqd+$Ik#4+GtAd8I{ zEGVVA+tnO^ofk#wxC@8_bE3Eu%K<)@YDmdL7SL3K?i^2i$wN*DVK847PLb$!rE6zn zi(#q7c;r9w9|A~t_GlHrNgU9eshvAg1H#_pfdU#!E%d8b0#dFx2EcyhUNl~bwu{{f z>X*FIPOnrXbgq@^Z$!+l6DR7~L>Mf#g#^@T#0Gs>epWoL_&$%Tl*F?r|_&<6d)GR@y+S@M$6 z(2Pv#-e(%L=PaVDQx=FglqhE>Eagyv2frGr?_>@ZB4^5xl2%bwTr`ChoG=>l@)A<> z)jz`;zOeuI=KlXh6|nOKixQSfKT9?W1?cwwA3c4zTdDuC_h>YF@|XYr=lFd5IQ+&t zFv;SKdPYgmEAo3zBQ!73Bq3;e{{?ag#9+;)8F}&$R4tNBSwJEAl313UDbT(Tk`>q} zKy_gpgSjMI+5)UN+SBn+S&J=r+ryT zS0)K@7v0fN1`t=)Y}a=hzH2siiErnevDM%}X zAGsR%!##gM^a^`&CK^FGSoM+W=4{6KZ}Axfkx@{~_?h_s6+0`^g2wI&E!}F@`>S?M z!-@FMS)5)($ytyl2`YwXQn}&o8f_K2-l`o3EH9!7L@}LJLzf!;E@--Tw;>aU-DN$t zw_3Y15X}iI&Sd7mvp5Yd)3Rvv*V%CGE+;HR^YyYJb~aqQ%Zzd^7R$08yV~8V!|OB; z&w{CtslbCQZ0oMs>du{U75%WRKizQcE+J^{OgUNV{Z!ws4P5;ZB?Y@!iBpz;u?`Y~ z%QG5WvVxzbWdS*{&LDNV-SL7}x9u!v0gJ9!cos%Hpm}u$7dO0fmkY;NYjx+&>KjLQ zqdRw1CYMQioh<82He08=FyUwDNIVmY_}O?~m`!GJtCl_bh7_q#n%Wc?b6l~`0gx0& z24GXJlRui`88WD4EiD?pbX$~E&$<~5I*t6d3IV@S8XIx{)g|tkD+VOgdvRP6|BPm* zU|uJ#=h_Ol86nBq%+ltK=T+@a)yQc5 z$Wr;#t;uGijOK-xPF6?MQaD%rLu;pqYWYCmpC(20ciuUdBb_nMTHe3D(mfosA)1A^Hf`lLZ=T zt2}79MAd+nHn69{Z>yiPkVLgz1HWn8KUDgKwZ<$z@*ZOB7QmKN0ZE-8>$JTsiR;wH z(n7i}q!n6=KJHo&PJ`6d_dWuEKv?S-fm)Xpf-SnQL}Y{7)}^jl;ndy#z ze2}M-|8?kGMoBo5g{6O{nKQGpJlx5=Q%M}Mj(tyVMiK}buP|Z@wvsMiyfTA6AvYw6 zl8_|@d1m(8ie*lP)>?N%uqCZV)#Inu4&MS;3MEA|wNz&Dh}>uprU)TlC@7u=6&IlK!hBXi zMPGxhfmn-l8n~JrLYCe3v#KrEEa;2As}7>J<5cbqnwBjW3U5uR=F~K9u;{ieH&@I7 zwH10)4Jdcaoi-KmFS9v#lPG7`G>#o}@l9SlSXBf_FM@@Rc!G2`qe(b&>4w z_IuuM4#>~=(EVtAcx7KHuzrW@Rn|QR39`~ciB>geuimIJoh@PF7$0wWqS3YlNse}B2x3z3xrr}Oo!wN^W_R}f<$RZiI1|qF!K)&fZFd652 z$>!VS3FdMby)pzzuIeMS40z(J`fGh5QoKiaR)D}#Qi*A5upV}(^EjJFuP(^x-Lg-$ zPP@F$wB9EwsosdZKRmYHHSPaFsfUvFCPzj9Tl`m?&UQ+s92?b$YYGKE(5 z)K(FY9rzZ+CJpSVM}XRePHK_0p z9Zc)Jr_QNtX2Gy?iG+)F+wO5N5Q1|-( zBXIB6UYdVc;nTr}4Xam`nVKOVOF}}7jjN@ilhw=0mWfK3${Nu!9Rui*8?u7^uae5k z+QRK};FZRZzN^;o_~`Wli=s~GWz4)fVV6y+yIL-F_9N5ABt*K`o+>$tbhCOcqWkO5 zu1&35)e;unx;GSqEta=VBct+I{!Fu~+`Nn5r_oEN68053uh8#Kdf6&sQRj)NYWOu7 zsah=*WOv=GRza7|vSR)^%B$$kSQwSF5&0ReYTW~de<-T9>wK%Axr}=|KD-TGn+o45 z2;yf|>`ZE_6>FKA+pfe^{Z7D+PcoiT~Pvw)3nW|FOTf_Za{25YI~SxU#ww zfopmuUc^!r+rP8QBHfrM>u_~E;+u)P-$mn~FLuqx>x6-m!3CsSxJCdU{ z9L1DQ`uy2UOZ$CJcL`bw7Is%X7Ij5HFNA1~L0bBjM9-{9zj~dls@mK_!TkyaWkk98 z;+D6KyWp$+Uvt@MbJP&Qu6A$l4gplFvRtLm7k=7S{A-P1q7{3tRa(Pu>*d$h?OVXd zyXS9IZ5@Q+E%qc-BV|@)x0>_7ay~4Is#%pj6*X5Y!uJ4K7Y#FCnmS_ldh+e)zg~;W z!lO~9>jCi5WxTA@dHhu@4;^c;QZme#St=E|9smz4k7YiS<+2{2XH_lB;&)v?>wb7% z*QYXuKV2N_etM{toU+YswOUm7*k4sosMVsnpMEMeyli1tDeZMXecAPq^tX21paI`F8EYxepy-?q!3<7TZeBAz*-S53nXO9UktJGu?4eUUu#W7Yv12u7tHC5 z+xs9C?YbS^|4A(zw>kf#IrnhoTVPA?f9=-qf9ws1gU9p#Lp%?3|I2a#{1e^-`}~<~ z`9DT6_s;Nf{XfLh zy8h*5mOuW*uZ~q;jSpx^fT9eZE1N~{EKSOUvx4|GMWFZ+Xwr~C5XH@M^NKN5H9Bg8 ztt&dMGlaEmJ~ay@79MP|w}=qNHe^+>N83(y%muU;-t?J-lsiqZFNQTH^o&2DZBdR78sE5N487XPsKsdGJ?=@jeC{Cs zSB?Q$BLDXW&+73XdwaW&_MZoNy5m3UXQ7pXKW2B0{E$1x?<4x-TZVqjeur@%v*ke_ z3o~?fgKS29tU{Zw5+-nmaFf~Bjx|{$HOv};7qhPy@4@a6X<>&DYV;%@nra9XKmW3p$8mlrUf)U%knf}oToYOAQz^Y@4$SPhO9eA!K(~@ zWMEYA)xUb=>`Z7vXJ(&1%hK>{n)2c-%9gg%X+qARpjT4P##XkS3f`*d#y_h+{GaVv z{ru-oSv-p_k~C*)_OZvx)j&IU;s7Ca|g zSz7ewFkMpfL?D%Zy-A`lmj>&YY+Im|i+h{bH6mPygH5 zAb+8Gl$M+vy?(S)_x+US;Wf>%_TDe2s;nC{lI8alv`e9j z#C)4v=TT9xghUDXbBgOei_41(I*yqHKy-FI?mNzVQ>}ZPrUfr@nsGucM>hG;MEc{_ zv?#Jszkd-G({c<(CdM8#aO_?a45kLWD0Buxk5(8Vxu8kO zg{&;5jC}Zs#WQTqNBzDm!q-!Blwbk9+;^Of4f1DJkP}k~=qQ<_j&pu~9;69RV+JMs zgx;AF8iu586hw8@_R^Omh*H4AlXZIxR7?&ldUTw#Q^v@LbN%wXF^GPc1|0r?|D1_d z{T}dx%*IK=G$@6F3{u`dgFzk%r815Q6gO%PFOsi6oV={@*Hdp1+IyUw!{yoCkMNA4~TC-Dksw|L58B$Nm37o+oeeJk1Zl zuyUNU3Cmd$F!G9Q9;QhVB_-Q*oJpBL19CWJ!R3!~+E2=u!Pn$7*b+9#Um2N3Lh2Bx zjqKxbA)pDF(kzoD<|sk?14$<&XOyQ2-V=(c1^pMFI{_ohyh*^>&vF)$Q&vC+zh>lw{Z>Zew@PPj1z5|>Ro=5t zQ9*_VAqW5a#c^yj4vK=!vO@zI^#?($6k0`ZSApUm7&e zq{roKb}=jbi*)1d@Yze}9lQ44vBF;4#Csj_ET(gC%h)(JwjuBR;jt$OM7*^cque1_ zrqWYUjpY5|G4bA}0gWM+65GC!e(3jk$%uzpyTK$t1e7a==7DwzRhfBpUU-*=OdlGm z^G;4NL*}Fu(}jiCLNn_ywV<9!ZjBO><{`_)LWA`!nBoM9Kz=lCFNG z7Jlgd5zRMue!BI){E{8#>8E)gOVX}f?W_4Lv;RKd8`Si_!EooX{?CIvwOGV}7Bo&T zR3+so?N6aNuysqI+2pbuGf&sV-~8bf8Tvc>{=gfM@5#{L9g2T$wtkxO5s9Ou{A9hA zEGb?GEH9!-#KMu1S?%qt*dBy!@w111y7mv{f!M>?BDEN@l25Zd{ZtQbrMfKhhz$Ip zKX5{Z$0GO%szsI=p;p;sCDM;eJ{8LeUErXGQBz_`v{EeuqH!b+puQt0`xlF_imscE zL|Cp-2QO66EQ_N6X4APci0=4wma6d{(djH`v{OT>lU9Y!F!| zU0EQ+5v{%sd`sP>w;3;cO26_ME$CQ7D6i@r_z*VRt)M!Vr`IX%E<*wSPnY$Bu{7X7O#|x&tIp(CCdeJ z?HYBfrA4hlRIwSO;g~(&-`{^W+<*CUxbyPm&L9XTlv27opfC1<{pS<*V%Rr*ZM+?N zZ-;x1V+>%p932e6mNK@8Wr2W<>tum4&T@oT=i$fJLiQu#x)7%JR>)2Ky7sNJK@Lb9 zagZygY<9(RL0MkpG)fAB1&ldVorh}f<9Qo}ex^bRokdB=`y7DuU=}QSpvuRCbI{p< z(-Nb>lw7j8&~GjIrDEauHXe4-KzSX-F$si9D7b|)rp~oNndf4jZHp&r#hdzcjm||T zj!ZO}n|kfF@q$(DDhgRhC>NMEA3-B?`GJ&8R9%P5)y=^%T z{9g49`+#*U!5-hG1>+-UgLq=RxyqQWZrC3a#p?n za+tG_B}GJIu2*QIP&O=Ukh9F#_R)S2aWV$xl$*5RjL2s`VMP%39Nl_ET>Wni3+@wI zViovBo~8i0G7ycKjBN{@3gvd5@4mF<8LP!;BNT}m*RZunwt(qyP={AQj2w&PGIzf% z!gLfV_NCc83=wtFO%16r6`PJ+=PKV9l>7tOrx0f~TCy>u8G{(D*56=@n1PHG8-Q_u z^l29Pvv5NX7VLL(B~7+<6eLJfOj_cf$v9RuHcO)Dp`^u><&w$kv9&6)R#Lll!jO6w zIURFV%p7CCxSEd<;QAhnzR6SkKx?<@erqGVq(`r9TixE`G zQeK!hdTpkTs&MmicmE|oJeuG=ak+)U*E)?4j^x*qx3Eze#Ys&qk^XC?)OnvjYiaT2 ziwBMU&!2^0$~Xh z;kig~b6LnaG}f+buv#0$bH)q4ja+{naYkZBuV8|c5~@j>XkyTafIec%1S#^8ZCCU~ zL90ocU(-AUZ6RmBm7)V9sHS2bj*i9W!=2~;z#sTS1S>yJ6R{tHElgo3VMyZJqE3h! zpoRlG0$9sL4@SSMsfg$t0n-%7ddUY@5hXvpe}8=H=zR)30{;#!8_VM-id@>A+w8 z2^{0JKQH29N#-m7RaF|Kt>7B9*%6vWK=K)#3r@he1o(oINm+Q6)k^(b zYmZe$-2N5K`+1r4Wy14n@J;O^dLIbkE~gFx6Ub{V>|4^Op3Jh9Lbn=DN0!KrDL@;N ziJt1hsheF+9t5jg(vEKNX}5?FEcesbLfGn`|gsz%MeK zrD=D;gO`*A3Xv=lhiaA=+*g}8vJ|0KV~oJ$5^(gRgBeI8W%=U=A`^l6J1m1k#>sq#@cQQ3+L}wruc1HdiYB2#D2cJOUA`$dT+4 zt5pN$xsK)3?gXQ8TN75$ixIgR_(Ol!lkIlQdsW(;d~jE7<>#81#r&{9erR?$B8H@#LS3O=cHi|)|!39Ev~WaN+}MaL7sAM z=&z22O!vG&d6l!8LZL$7v{pkvt3*rOdg3TLC3OoR*m|=d!M4?SlWc8qBbrIlB9)dah!%Yke||PcExfXjZ=yvZ9EtHdkwLgB}ZCX#19w_ z3`!`q&YF0pg>xPauV@kofeMT*mmVC+bDN{BwM=;h7#QP(&`u||+sRj_CJL}594=4R zjDp^x4DZ(14B$;(iRJ?L6WDf>3$*kyge-P>=nRx2vH$T%Z1_S1mDxp3L$pn4!?4il za`9t4SK=;@E~Z6GcVZOA;nr#pm+J$XkI2Ar;QY4a*j`NYw7i(g<9ls0kf(3MCR$2w6#Z$!0C$3j;^Q&RYtNQXCLtbxgK>jVns9HAI#gO^3u%uiFF`ymLSP zQ-)RiVV)KIzb5g^*_6c@%eh}<>$rlK_S*;@>d&$~N{R__zvF%PRsdACC;Q*dhQ4ZaidPfG zVERuT^{cip3Xx62&=^g@A1XE>2tq*}W!n)=;F5^^*)Gypq(toLaVk#ZfiivaV9LTW zR_=(DGaI7wv@g)K2@G#N@z%4TBYi!2xVFPFSyQO9q^4{S1FK`|okyYy5dAbwU> zi4k$%{8rM~tx~H44Q1J`%rSZ!57SvD^mF(7p+9`)4P4?M(u|IyI4UB>{r@oDI{(j6 zq1<``&X^UnmR`oh{OD{aqUFLy?X&)8^io!bM6)cO|3{1e&_ zSOn$^D}#yj%h>4wQCT&aSU{&Nj0X~KapA#|Y&E{)=EG(;y59yMTu;2KGCQnL^ z9^6VXcnVL_7#EKyJ6eAX1Cq5DcHYbK9|mx(#G2dZ9`sX#wy?4WzH&q z+Gh>-gVi9cF>@8d+-~mduA8$LO^8(^qY0*kDn4-=&0Mn8@@;E(^=-v#>j2{=T|60Z zx4wBQP^8aMXPPRq&R}pei}k`0ry1+Qol+fB&QBV2J>?{wY%}H&y9xP|9L0 z|6eT@g?msc=FasWp6}KDe|x(-kNJNdYBQI z0e&A-gA{eslQizAL%5pBPnT+_7JcZlD>m}k=MaCk*yWk=%pRymXcqlBPs@yt$cN1g z5@Bq9tcH7t-$NDZ3QZ!$q0tq~$9x3Av>!H0&MIGm>jUrmtYP)6>jHMQr*;d^EzBh_ z@+3`El*_LtZ^by>zKjKPt-*b8FIeGz>~bC;dsJfpl^so{sbBevHX-bJ-W}NSPf-#^ z$;CJ2$~0ytY$AGAytZHz4k67owKmSYgo|C0`&#RK5poMkbXR6+bJ(;^-&$qIRz+)F zr0zPUVo)8+VYS8AK@o9Qkm(^>FHcmRIcw8wftB-{kg=t@+QMP`YcfVnimbCQ*b{`C zLjP(wW(8G~b)1H$a!0`INq;O3R%M2=Ji3Zvc43EKTG6mN)KdI*bbMHCbbR#M{E(N) z0soaVS&K*P#UrwLB7Qwc=9~2_2#xo5-0swchT81PK9_u4yOSYBUE^lL=hZ%MssFoQ zV@Cpc*<&yo(pc&rHeG%$c>%P+aub+Az*MA|MahT^^i`f2{WK!`dwaWk4j~hcd(1t7 z%NY%SN#glQn!;5U4EZwij&($I&7e)Z;dF2FDd)l?n_$y?9p#(q2Vg;3OvMjy)02fi z$)D6UYg$Z?A$2{>p7BP$V=UBnE>k%Mb00~ifjB*Vix-jb?u@*o_|yCM$1RZ&t#~Wy z$%VA+$oY9;A&)zq|5nTiv@HM6U|7%p_3YW;vHtsmJm2R0N3)Cz-MZr|6Vh%2z)jM+ zjc;4LbFp^c{|M3gfLm zR9U1>(Nlt~RGGU2A$8$0MJz%(38wCHnVMbJ?XJ%Arv7TRYH77Q8P<4O18^H->A>9v zyZC%uX{pQUzG$craM8oQEz*H&C1!b5bnP(eh1l8+mmu4wSc)f1IO@>))ca$U^Wv{j zG5rZYQT5y!6{goby*4WMAkRa5m;{o6@Mid2^&u&?=!nNn_BF8$I1uCBRl0Bclj8MU42 zq>g^cnHQ^Y+`)%s?Um`&v^{`K61bHPd|fXs@3l<*?h@o-X&}QG@vVV>IKvj%_D5xVgB5i^c zqWu?-UISHfs|k5gL@)-4+5nZ$;xJ7L_NlPx0_K``X(EpGmCabc1X~Un4hDldo$l#O z55t0!_s?RX@aD`bxhp>O!YH>;oQY3xPFuL~7Rychd+fIMo!@SMhW^s1R>Q%RpVKbDM0P&y! z(`s$ro&NIc$>Ez{44KVr40A|-%^w&}Uhq*{Z- z23ez5H^fUeUkPB7Gz+k#t#~$=>@dwB(}Qk}xnQ2udlG=HAKxFG{Q1p$n`eWp<)g+I z#kMps2k5smM$bfn&aWb&WEUirXWM=a=Ur8#J<6{qe;v?|m-5ddO=7MfDkq$5xd(>!UBzIN1HcenQtG}jHh z8Z$?nB_^%~zTS3E?>&dWNd{ON0N2v)p~K%0g{8r7&1OpwT_bX>RSW&22Ch}T@pth$ zBUzS2t=Wy%k6Fufrmwq2thMsg`tb$z?^cK|Sa$gMLkVrIXs5Y4*GcZmEWX0j?cE`E zR<6xTgq_Ur*V0Vw7qWu>h`c|^W zKCj>Z*dGj^J?eiC@!akHM>P#y^BPDaO{yiOoN;xu>Qb5Fo?@b~yDeUY9UQ>OwNpsp%T(tqR z)dbl=xO=}gqD)6mbdA!{E8@DOmaj}O&k-kXlw8p`vhsmS+V;r0U-+s+nd+s5({+;t z*C1;7l=L36rShI0!WCv+Tf4K5SKmQ#n`YfH*~2TLmb+xDNdKLuN%v@a7WM43eR?df z1dNIv(Ls?V$#3pQjrd($)9O?tQ~};q^oSKg_2Y;atWqg*h4}oH)eIYfrk2_&;{i@FA zq6yyuA`jV>Y8nT}M{;-JR?BEYWVS-t=~j`R7EpIwCcoYw|Eq>qrHR-?EqE$NTiahU zMe^A;dBQ$L98)$+m=%n+*l)TC46TBR*)4Rr%)^t{SRxUMHzpT8Ifk6*M6N_~Y>LwU zqKeYOM`JcgbKS05WzC1FTROIHl$!4Gd4T73@;~F5+5_$`{|EbnhWsB69{oQL@_Y^X zzm8vj!#;p@%5(oG%86U``}Z3je@s6M5GOQ>iTi%aq~@zjfQ2dJ(Akt;VQu&Z7;lw53+oW-<=t{4PKTffsJbT&xBF0iETRt8CwSctLF|GVd7UFkG|F9k{@V^Eln z`Huhk^z5`qbLMQ2Lnxa<{&ILq!YJpCe-RaZ_%EXG_~U=%efY2XVS3RQ|51PPtE6uL zjcFhx9t31NPknx!IZypDy>y=X#Z3G=&7%wF>3=&Lh~|AVp@*j_@-(It42Mnea%_HNtspa zQbRpzU0NAKNoW$W(sU<;9zOkd=T^)IGgCIntz zNxps^R1eH`Iim$@;s&D)Xrga}Oj(pefZir2i7W{&28n)Ys7qQ*8+7o=0r}8!tZN+x zo}T-eZtkffpuL*^gdCobh!Yy~RM|aEadK^ev=f5}B}o|{;Z7o>!BotOsW;Abv20xd z+jpE_#YvUW1P`A{!0;ojH^EjC6;u@h7vE0OI8LvzL^?}^xaA|qBY#UvlCai%C4drD zCA$TqO`$5QQ1Zr1P68AgPs@TwAycTs#hfkXU-;&p+&L?P@La==1e;B19ou=~@NZVeBAI2sP6{K=ZAXgD3R7Ix{nX*I zEoAId#)1O5CS&;|&1WcsJgBK8F{sNT5W(l2N7ZvM4+;(ibKSW(vPR+%$&nmc&aQ-J z7|(UXA}`sw@BF|2^M9Qm$kV6q(t?eiK6QQ|N0q~)tUW9A1A+{T~kgd=V=fD zXX0Lw%*8>}#QL<%Oq7rby%OXUd#n%$F(WuY1V36dUf-!>Jt6b7)YO}&Wv&m^ZCt8O z#ej~OZ#X!kY&aJuN(JhiY_Thr2#SghW;7{j9M4G*Gnxp6K@YMg_$Vr{jCah`0sbce zOp*~9Wt^v%EP=#4%+A!I@a1JNMdD7%9IEI74BRq7HbECcY{O=~B@s>}JDzXXhOkXo z5%@h{P)gHEs!yC;u!8WS3@)|Z05IDo8R8tI35ROrQ>LYjSeAHu^ctj_A_l;%j94a& z68mkICW8GyG%VufB1Nn7Hc~|{fH;=C)52buAreJmR0yV@(s*K4ztX?(fEa<&qy2{p zCEVk+@mY%qUF!^52tKI`v4c?V8h4RmQ~~Mtl#VcFLU4rI$kRC!0Ilk zazoyHqO&Y!5t=d z+@Db)F8Z(2poEJ>0^>f-B7YWcRD%oK&Mym(E0!LvrdO(~_IZWES~fyWY)t0tLJq|7 zDAePYC?FVFv#Cgln4l~TxmnX%`H&GDkcJ4pLj50ktICoXH-VfBN<+td$Ph3wBL}#y z(}@WrlgfgS;^Nd3hDa31IZ!2_)@?zPeWNmh@Qek^G*>-|x0+amE3AkQpD{W{lH$w9 z>acZWky;%(xsX!(<){Qfkh3IY;PI8- zF3_$gKqQQtROT_lM6V|!OjMX)wa;K0wxs1h+Fz7@*sSO>?C&~q`#ajZ3H-! zoQj4J(O2KLoZOJ}&tD|X8pzpewqi)B*eF0-7H9c@X0R=!6LSVNjAbn(c@I&J%w9iL z-nA1H3T%wfy`Hk1St}9eA}hyn#HV#ziYIt7nnZ!(5?`oR*ewYCGA<&qnHaN>q#_sV zE=fbB;>h=wX#f;vAwc3NfZ0Zq_6<3Ixx4?eJ`R}+QH({$#8z}1)j}&f=|t}kI?U3} z%A)tNFJI(!I=BsB*E+||;L8`EUj?e%s}BiOoA(oBsJMG>WYM|7TGA0^jO*4$XXk7c zBRpk7K$1yJFT~U$i{NaTOyPoGbB0z0DSEjyGT^#Am-`~9o9EirAv#cps0tR9B!!~Y zn*>#uMiL_-t8F`nl-pO4ka;SF6jx#w1f?7W?DY%#uU9G;i2Rs>4*Z~aEJ`CQa5|c( zgQi$^+l>_^jgw6~46sMd3?p|=V|}smL`~WaKvJAqJ(e8Tb-hE#Rcq*?tUk(S#6s1* zv;%P^39A-`#pGJ2UhFF5D|vGuR@FyLIK)CitI9PC(d%^^>{^PNYutHQE)9LA3eY=|kQx;q_o zhbQEXOnIlk!s&24x0P348>m2V0aR!v&UqbLs66GhY0cFfquR10hAI`9YBp1$xGMvG zlyp(|s~{1ud%yGzGG?l$uyyBcmL8QU_9fuU(lMfwN)7 z_~aCir6HR{iO|_i<2J1(XEa1#IV=L@+mKlzwW4hu+wVXK(Y{L%h?UIURZtua+n`~B zySqbhhu{+2-Q8V-6Fj)PySuwXa2?!T1Hm;ovy=Dzs{Y!4YqxgK_GC_{rn;-Ar{{X^ zhtc~^msW$`oizMmg%%<#iZV4B%gmfWC@kHk>J#F;3{b7Cc&OKcI>|XBn#{ovkcUY| zI#V3JfK3Az+Uij(LNHZQ%gM!e{+W5!Gcv)EPKI;dB1?zdpOwoNlca!Wa1xR0@NIAu z|0dII(2hPaiHih=SAfaJW#J0P z9;5kWGhAtg6Mbc!Bh2+`hh|ll_hMJpv%4TTN4}Z z&P`l1isa=k+>-pSpQIJ!=RsGT_7L|tj3j znzW|E`#5LanQ(%cJ{ZI34F={VTwed8;2*a~Fa-gBFRh7<%Ri&NP44HA& z0!K(DLBWA)_e0GkcbjuhtZyliBy6J%SOrCS9b0ugS}HZ38yUv+ed$QJU8GJ)iNiX`5)^Jo6ypiBWZ5jE-Hg&vf@U|N1nxApW>! z<5qnnuL#q{cVkTxIq9qXDVKf)0PLi^LMt9VCJ_jh#r1c*mNW zULQ{75`Z#mC!hBX;&kv+F!LVP`56r|L}nOE5Ic?5bl%I5lLFosv^uKvS7H+bqFN6+4x6@-+;#p~+@obiL{PW9F{M>*K2bE@x}> z2^jXBbSZqw{TLN}HgM#3NF;vL{0HXHS$KQATyYCr@P=N+5OJ1ZH@T?4R0l3%^5FxZIy29ThzDI&sh13+tV_@fO`p;0_WeTp>Pu|Yy=m+-MIK@i4R`dma2^(_)PCb{GhFZ(dLHZ?Ge<3e3Y>90) z_Ocr~Rzx`pZ!$(RKd6H`G9=im5G0$5aF`v#x#eZd|8<0oRbr6UySJ|oXx9$BJ#sug zn#>VKm7^(f^$;>VhkBi%J|C|C_Okw+?|vKkk+2LcCt|N%WdyrnYCH2r&O9!8R4S=9 z6E#^cQG}W;>L|S3LBQ7ILf7(OPmyFgwCN^BfpmI=XqxoL<6y36{v?b*lu#V#PWUFF zNtmB?yIT&OkVUMqO1vYc|7fIKl84#ymc~K83>8WfbX0axcu6Z^HU2eJ5%=OHJi&-- z^>Xa)Eb=_EbOd=8*=Wz#XfJH4eB`yWEbgJTO$9QZ{ix;^W>{uC=k7Z=HW=6jPO;X*!9&fFoZVAMEj=eIUju9$9--_SdANDJSavO5AQAC0&t;rvLdB z3S!x6e8f?3uXc7X@0U!bU6vN;p^bzK@4*LNnBy0Lf6>z}k1$o;Vf>I;tUI^;SuoiH zlQZwH7YAyy(OsLb5TF0(K|2-a$`Utwbiopv}ZoHmd|={_#j07}c3N^U)GZ^8=B>Sb?+$ z>_$XRWh4AZa1dO{cg|B|3`neF?3wH)7s@~Isw%soY$)4wUV$H&Y3#$Rc`(*^B0*F!ibSR>kks5s2R6ry1sSp0Um$Wl$BVwa_z)~^cg-fY8Hf|f*s&2Dx=ql<^0?vQ;0GA%8VYLu{HT%F) zQMN;w_oe+0DDqmh!+P2ZRJR`UqaXI(sGt+04u^nMO?@wx2F!Ju$en&?YPaz;t86;` z6MNDu_l=TteEO8O6wh*cofFnN%18w!6v=tOYV649M@)(ciZHse$tLp)2KR{k1O-ho zx~2HbxK!Y>R3C#5_62o{^O`DStjnQmxyI>nYWoaYhdSvm=hHFV8X8v3+FnXu7Fbuf z5C@KInevT!TA|m3NOE>@WQISjpn;bbuQn0qn{wRF2gsj+`RtYr>x?Bhn|xAt9klK6 zSrKr!3k_Oo8XDh1jaLI~!8D;e$1cS0KZ7>NBUa%Zd{bjkS)3ZwtJ$H0@W0KzVHh4m z4@l`|8E{LhH&DxCbCm_!?-w&Pe}JJ$#hUEb5rzf3CwXxkf=-2*ARAz}H$@_;xbH-W z$)ZgnnNqcqn_nEObS!0iud`9&`Dv8|^oIrW%(vY=Anknllfhj!A z=C8WY&@p9QQ28;-);uzQJ5e9?WKc5f#F21`ayh@>%dImy+MOCVW+Ssc&>(tj=_koc zSFT-~>2}os+hLII>nfB8q%|jw^A8fnOHb?w3><_WZv12RJnw_VnbBR-c$y#H7AGeV zQM?Lb(qtAA*}T+@X@eyeBx`67?&%2%8pMaR$DtpmTC0cZaANma2om-vLy9h6jOQWr z;g~Q6q==!TRF0f~)ci;!RhU$&fg|g2aYy3%4ie-48X7Obh; zFF>tU?>koE_O%?sF=+MJ90e*k);ISro~+0&xy?{~9EJCV!8jsTqD-jr!iG7SG(YEY za&OE4G$)J`*A6i?=wcQoztm83xo@d)tbUurYMl zSaa!}(j;*f3QPD{;SU^bIl?C&J^>`ncDWt>be-)qX;ZGBwx#YusBr)wCR4>G^8LR# znW9*blcA!@Kp+;eH#~?$w?j%Wk0fuAVR-12H!|iZj*AN*?K`#<2iA>$^5ryl8gahF zS*xtwrTo~jeB}duNp$`FBifMUa81e6fv|jm3s}{PNF4*I<%QwFvT_LE(S5ZI z|6@Y*+dso6F^McazYZ}~7b5%b^LS4+?Ts_kw519>M}KWF|Ol$gxB z08_~rp;j2S=2t3JWYUaW1gK8ig-Q|;#K51)HN(}kl=}Mcn$lCL4^1Hvl?_&xJ> z>%WrDOH-itbaNl6w_KYdpt*A+dMz3Hz;O25h_R+5Hh-oC5JYK6Q48^K5;z7;b2@FD z(t7T~mcxaF`m1sdhED3JJ zZsG6`Q}0vFVU!qRxue6Sl?>I*s%BJ$JM5veE^+x(4?Bg!Xe~6~ZIL=ll&@fIPS46k zpeNO{%3@g_3UKi1X$9opkhL~I_HArw0P-?D&-i|Kzr?=oIDvln6Xp^G$_aZ2$*oi}}x0J(Y%J|gsq2mOA^ z?q$KlBUwXSjk@lemrE-9`tC2Al;k(*$&~menV?eu1k~xc?y6{~8O1qQ zZ^3v{ViC;Bvgf*dS8fHWwwA)qCCb^QF+{=dwcnxh^UD&w8?BpgPJ5cmG`ivzIBsch zI(b@RlA^!iAc%v?r{?IRy;T_Hu9G;TnW#<}1P$qVBVR(4pnUf6-C431s`qF)4q9t- zH$4m$4IG8q^O{!6soEMa&hS(~C68O({(Nzfza7NS6j}jVaxq%}T@6 z4n-NVtvbE-7qz`L-|dNvx{ITzAugA>6rp6k*(}HDHj);jN}~F%aW?7Vs2EjXUuK-D zG(x*?R%Pqxh!cyD>i)deH%e7gsx8zXA`FVcHg@20&hjqX=zL}(i2hVL(nNmPdb3vb z@#pzX=cc;F?P^h3)^WZkwlovkVeNZ;%SKJwCSJ?(z3oM|mUU-1!5Pw)Np1}fBOu$x z*3~kE)p}VxVN+W2F%fVTXv|oE|9=23E$C1Cd5x)lswp4P;_fSdfJ^(z?J(A9*anFD z&Oq<0XM-(Zo8ab}=RHth2;!~W9s|LL`UBjyj^Uzj_ZC`9VM_m0eAql^k_Jw4h>qoK zN=sN&XEs3;*1*SL?NAa{=i6@5<)Bw}3=1eFfZotK{^dr1LJyzO-E&!H-7DB)F7K{+ z>M7i3b!cBYi==9gc>hr;q|>5N0)xHNJm?Fn7vS_qM!wfNwvlI+dK+>LR|+|8Wrou~m7%$w%4GhVsYh@BXx z;!qvTxKC3qRKd+h>_$%KtWuW#+Hqo>tp{6ix$UOhQ}_BYm(9T!_MN(h&jFpAthh+l z3=b$-OoD0MVC#1Kb*M#EwfEo6uqi*Y&1!mD+%JTmUtL^W@&wCqU$Qo=7S-{Z=p z4W8qDJLDS?sUohB{uc9?poSm5u+TF)*nXs2W=#QG*K&=H?X#8X3}dk^Hx z18R)@%U!t3bAW$BXGqTc1;_5}r~TO{LAgb~UK^^<*kF!H%?<|}*v!u=TXywn0|E|( zj0V70O(WPj+i2g3`EH}<@OCyesjEiYgz4V**~X)03Kgo9G2Tss#S(BBC@(a*oo)B> zS6CGgercwX;cuflMCjR)F{wZuzz+PqIB}aa@h^X=ELp%`+YMIjNivQp;{a=8O-0_c zuaBA;O-bb{M(zrR^5FpCykRaDqilbY z(+1`f{}kg1f!o`mDAwzP=f+8Y64Jl2Njfi>o}Tpb*jVx2!XEzY&R&ZYuAa5vAeA`$9kuPtoC&o_`l#$sMh#-jQ^7TddH;wTBfyPzP9q|L97WrIM!*!&tyEM zXHgk`!p&hErCt2SGjO=*xsfwP7G*7$QuNaE`*GYiXEq z|M;tAgVJ}GUn!edBwohJx-=r5blD5o4&B4fUmosi{#O8_LFde|_}799iTc&yi*Ypc zJF6Ei`(-HXp?z(EQZjmXGIat+D5Z5$!~+CbC+lhbD_|;bF)7jpWVgoL9Uvk}=fnA* zeV%?DzOThee%s0J;d^=pq*;iz-#;GS`FB5W3|P=Gf7%y@C|z3b=6gR{NgCt=9bEZB zisWh}?#BoiWp)g3xcZ2Vp~Ov8rs8PoSR_>m(S1s@)uxxZr`iR1`}j%khSARD@A&Me zr<14EM%|>_f^y5wtXu^7ui{X6t%22ZRRo1}ldXvVox#v{ zK>uGmm=*O4)I0pOPfqW;eA^F-rx$6)TU*^(+s(o^Bj@A*lvI_TXX8mtS=zn{oEka} zx2={+BDVw28k55K&X__FAUngFOxpnXWB$;&FNg%wA>tWS^B9u<^l;A4;>`? zv(qJFhw-SzGq0dbQ(JA6Gn-4Sy8Y8|`m}t39z53v`%Qm(RIv^Y^A{2o0jBCxd>@|= zxZI6be`O}7xDS1NVAt!pcF)Jc_Q$2^dl`(U!!fYsk!4B*y6joJO@V~`t<5RIHv^@? zOYtL)sFcqxn8kH(rsce2bW(A`0+I|0NxD-%|9ev#Td8zr$g7DV=AZfz^re`k~d^ z>xu`mR$HW3pt!nZOEj#H!)Aebz%^#Jje~qer)5^qW>;s4z#pgXOQN~-V&<0BdwPqU z`d`&^#HfeY6`;5q4Lk0F6_X%Tg&#P1U^ zZ?i3CpQQr1hxviuo#TPFs4YG+UU6$Go{A`Y@J+QfTWh0LSNZMoO_%BCdnrnj`i<(v z!0ClzZ+3wwFZcDY0;!td65??u`D2NPT#rQ(Z%nn;Zgl9CU{`crL+}MJPD2P96__0& zJ|7b{p)o?LGfL{{*c3cV_rJBz8yjzLo_y{s?cY(vJ~O6MpLKZ)6Eih{GtB`*gWgBp zciwwO-xNW^pJVc;g`~QyvSS+ArD^68o8p(kY3-?o0IZF$vp-&DV7&X#*~EE5L;x9I zB%e>xiZ>-2CC?94ym?VnJ_zCbW}21OgoGsHTOqoPMQ{mg@aAQ@;$AvJTTU5oUO{hl z_y#FdCKxTd6@>Qo{^OYAmY|-`E2fj&8iTP0%eqP5Zz`5%cq)sw*&{X-MFZ0aF`Dgk zy{AK5=R%AQ`0)l>nb#;f83dA;H41=RmrV)>NvXr{)m~+r&e7vfxf_$fXiFQE_N+IO za}@kr;PdVr@Q!q5B_4S8^uTovp;gBJ=wF!KuHNV0w|6so-vs6)Qk33)RVHWC+X<ZB(_=j5t5iS%O@o2OT;z<+@v`3qiPh|e51kvcs zJWB4JeN7u>O1{js?pAncl6A;XD(*T=Dz*NSLsifujP4#l>DI<3RX7AH?tJVBketbs z!THcCk>dpo_)K}ct&~}oiD!NRy@|*g=55-bD!K%Z$6HkS$)G=)?G5S(1-YszuuNkuAv8{{)TJ-d>hFNY5Q$@^Rlx>a(TJ93^cO5asv9tTNWvA zZa5*U$t$(=DaS6oEzOXqu^z_hI*J=FUKxrFHxmxt$S`z*YSe1xzJ6!c15EjTxjEBAa#3rZD=pDm5GnDI*gYE83^m*qk zCr&?emOxk@LXseu72AQ%Xru`P>ycML*GAVvM1KeE25U~|JlRcxxGgx+TH!_H7kK=# zwW%sD$)1+jqlWZ3>amlj5~rv zmWaAW!C_2BHfT9mZYorqNU2g-6p!vy-!(v>!$kvfuOpNQjx%eX-wX}`E9n9I0HPYM z{u6dSm_% zq}1#xKKWz%_|_g&!DxAWI;o>{p;~g8@<;_THuDHVlM&~7qcNGzFJ2K?jCe4%j~z1D zzP)DcCd0+%#`^68Sxwn?;$QE~wLP4-;;qrxngXE@iUk zE5 z9Y;rrHAg0eB=|+1H4{w+G)yd_DJzwE?%X6&3wAX4AYwDcYh^VfysPvIh&M&>bc)Yx z6Wof-+TanM={Xgmd!Xg+kwZDjJZf`Lxb5bFc7aU6tDyidS@LWtcYaWv(m1Q0{Mx%W zKe;qTEswjy2x?CpPtT_7lToWY+YpZw2J+HanVc*^1H;Xq6AmdZal{|Gcnxy6lz%oD zL7%R+isL@6SN%2DM6OKyg${Yw1zKCyzANWP+A$+VB! z9yDR2buQyIqxz`en7|Zla$J%RmxIq=(GnR?1@FSPB$T;OL7IH&pC+TX!qhC4@U{nJ zpN-EsSn6|c>p>~3a#)j4R{qq+o~c4oK*$VcN8xe62y3+5)!tl+eu1>X8chsm5v^=k z#VMe`qYoZS=F`z1OgTUe&;O{T!l;UY0N42R2k9p5R)d@CP|HooVa#?Uh(8eF_%O8l zSY|x*}v`uhn;%`f0U<#tls`ir%uL_=!=x&k99_b@ffu{p(G3z2I{uJ zK=Ve7BIv^D*oLz1!W;MuSNS%KU4eHacD#VO>C3ycWA0)J zfJ;cS-=|dEj++c9FVSiJN4>-s9Bnr%vI@gVTHTn%iw2!zFi8f#292vd{&|wo-|d1$ zB6|0RWx{YcQII3Pnc^P|o8{2htvQC+#F`8Odh&H@niqHQAuV>nnXBnjYaxW;2g>E6 zjLJeRjTWj2=&Zup_|8qGEw^N!xk-`Gd_rCXZ#7aVSslQ6aFRn~pX14lOeByU^~*3Y zq^MG$$cBe2s1Xxr;$8A(glaYdsa_?_A@}eQQmt#f(7$msz-wxuk$g3Ejiwk*i(4{2 z8zXPwb#;x%^dV~(9+hoE5A>}raB#krz@Q4z&HfsR(zMdRk29eR7f5O;J@)f}UO+!z zUvbP+(rSr^(BR*JcE|jn93;#t`vQ9?Iht^^wtMA6;V0BJ=}QqQe;)01SVtNjEF`kR zx?X$G;W*Cglgi|XZ(6Y%j#p_={?W=XeftSuq$@@J3(*F5Qb+2RxR5ZT$FH*6mYN20 z6}P}!z?`Q^gM3c3s((VdNXmTM6i8 z@Ts%hagE|K@gy#@M_x>hA~Kr^eiU=JcrozQo_U4ES1r$qYx^bx!b2*(tzq_M&@~w0 zqsChldREoPG;e7p|OfuC`!i2&5g;T3a4GvVTLvVc?e+r5$p? z&Y!cb)zbR?-R4`E%g^ZP5F6t)9GZ+|akwxa{>x3@*!6NSd@y4A3Ylt1jQrM82BfQ~ zI$@!0ay%FP((TggD_@phK*U-rMxnQhF_&XB%)4m3`;9qh0P1{NX;P-RfJjl)o5-Xz zi_BP$R6QfxGJbY@z1pSBlR+bovbAGCS`3tL?{+E`v`dw8UDsP`+HDP6raf(w6d0NW z8ouIX96T>S+(Sr+&&vLWmT^cqcPZ$>&x<=3Tq#4l-oP(5BqCzL7#L9JTw)8(iX0ML ztVgB;+}p(LA~Tf~#ukUC&MXUC=yQQq7|W?j;Nz%5Sss8EoO@C7JP9kJnwZuVvh(uxS>N~4uADxo9VUrB{$ko!*dvW4vXX1=Qs;p)1W}`jo(TovN;3y z1;dt&ST;v*%~Lx#nCs=SHs9` zkH5qxODyw_R053)_q8c>sS4gWM0}~ZlFJWq6C~M3}F0&;^BE{_BNOD2i zhga>ul8?q;`!dvEmATAE*Zdj~(%x-H)ov%9rt~nE-I$@I7R0_MuF$^%-Zsg-Ho!6>TaQc%Gb>uO`Nuu^-HTLAC@E*GEa%E)Fdks&E45b$jWA@JK5OM5cR-Pm z)}MM-AqnKKW)XjUQnao1&62Z-dLfSk-0`f=T_m)?vnYdfpN!$)1HgzD5D`HqS9(-^ z8{^IBRm&(S>$AAiaU~Pu-J`CtHC-^&P^5Ve#~lBS+bmc!fsD6i(wG3 zh6D7gE6Rk>>?p?2)=Kr+f)5b~*=!c%2J$SA0z=*SMgrOJuDha#V8KGTevS9!z&O;m zk0%CDo=+9h7>F9^rZIlE`oPcyyn9=_vv~tvKYx6fz8}MQ{{QA+SZbBa(k<Cq)_Amd=Ih@e2W?QcApPdQqyg{IB?jZa5 zoypJpNFWqg!Iem`e+L9=(Wq*&$IUwJGzSX@7x&6u^2PopMrWwJ3smYHu{PGHR!#_i z$rt4jo)k4EF_C*aIAd=&>kyh=vs$RZ896Xzz%XSe{L+9UKycaT!?Wr6>}-h&i6KL~ zMKp`!-T!e1=goSa)yuh!+|xEFFlsGkX~JEb+#t@-)Mml2y7u{qb-uWY3X(Fdcs63n zgonIVcV=Dxb$RXk(?VF&o}Et-MtGA`*caH{_xfYhlUv~#^4Wu~W+R_NJ5|=`{z%;gTCx76*&V#6Amnhmz@@tZpUWe2a$jX@WHM1s28crGa{xJC{A(?mt4dDHjJwr17Gm{jLx{PR~QZno&fgu)%9A)f1H+ z+tIrLx5oNU`p1iTuN8m3ldMhuH#!c~rii7X;7MyH=~dK$mGt~bpT<;GC=CrJn?J^pYReeA8q0iKR-Vl+5;e~N>ka3lv|qXO_csA@n}hIRL&ccCM$}_7DE-I<&6mnkkEbWUvF!avtl97{bF!!fB141{>Af%o=!?-gjG$p z-;IK!Fs7sdC5gSS)Dw0+kX1_)aaVG5--W0q(&*@T7f0BoVW1wr!77TE$ocGNY(9it zjGv||8Y?*TH~KcH3|_B1^sFgp&z#0cQw9Bz5DBBz<&_z%`v1!61i2bU5=&*8D#A7q zbLv9*wHc}ZlhN6BuY{k2+Xov+HW##3acQo_P2vT0b99I^POCe6KJzjz=)y@gHhD!_ zk7P8pB8jY%r!NK=2Z0tuIP>5dnU>6o#)~E6K1O{&#yTaj!NFGyX(#xxCFXO?Ge@P0 zze<`ymS$BzzXlk=4F$hmzheb`X)_+2(B+upo>(7e%k5^KO0_L z2rHie5!F+33Z+!DT@$azPZ#PS@Ts1Z2B2N-F8dqm5qaLn0ls#?!UTN8l>s{?Whu)T z*L@fNXG#E*vp3p1>@pUK~{u9wO=>GlitqW0axD^j4E2uh_nhg0U#krv{gk#unQw&$t>GUvrqB!jVE zIvM2UZDu%-FV}Nqyo8c&5gbJxLMegOTQP=44e#RNet^RRO=u# zqR*!YUtm+S6_L8i)*nRL&es2QeZYRdUAC=g0*vQ0t|*V01TSnfk2+u<3V2I4XrfR3 z<{whd#lJAX6Bnw<4i_vmoV6uq5IvYpZQrjDL(@aE7lOJTk07Qx`%g6HKE#q=`wP5R z6kE5hX%%EbMmi5xFm>$PIFybTo(Ogqzbo`a!YYu3G6_{!Lk8Cr7n@g~$uIg?X7gxn zxY_ErLi7OlrVi4+v#-z{8Xh-t_D(9{hPn(Te=pa(&)&FrQ$*@&Wi)E;mLZ}6BB6)| zxWElUUtDz*jr}vRt5>r6sW8!SRh3?j&93_{+TF&>~}mw=pfI*=*v=>O`qYe%N%GW&l7+@pYGu|W%N?6 zY&|0zord(w+M?biY>ECxYur=`)$6cGS=c}&C5bf`u_Uyp1%l@mJR!P;r=g135eV~_ zLpIgQWRR?xLd)5z1zQkv)cr@+6gk_6j>D>>FaROeG=2IZIf8f6f} zy!mk z7}j71)U1!mM`^p{-Qz`p89&kvCV$3=gHlaG9h@$^(7E!5>;KXk*8ie4u%aF|zeZSe zHfV~GpEH<7GN`W1j>zYfBm#`BDO|8`kD`7XwEVhaIrX61G7*w7F@PffTa(FZQPlu{ z&X>{otp~;ceiuTCB$c@`h3*BOOdDfIc{B-)qr%KNg;#0plBqE2j>pbxLJ)M*bz#5s zQIW&-B7MB1xe1JXedERT!RlNdP)i{%%F#)Zpk=XATGf!O98x>rKcTiOVQCW4em`Bf zz0urUC}-yClr}O4hb^MW`RHh3Tl=iERM`)T$n=qCS00v8BV7{f_>-W&ta7(#GIvNe z*~RcW*(1>-_EQ6*pw}7sd+6<2tC3$(S1VJ4^r5t4*1J7M)NAz0-e2mmkL}EWFTcBCKUwpc>D;_}eu8bIzAJj}Wk`q@UuFhx7FWz1 z*}!x&$wnDk5O31zbKvsrT}xpRjUlY^PihBgWXdCU+wU^ttDH+s@-q)|Hg?}cBk&l@ z>Y}+Ysw)F4MjgOcvsX%X;oKwW?-o1hA4HmRdDr>C(qDgLAPr_aef&<^(6&x5WY|4J z*ZlaPh1Y}_7^^<3jg6+VFvQ3-H}PQT<0!M=kr)18eC=>;OJ;X4Z05P;<kta}v zXtu-AjiNzd9FtQ;`pd!FQekUTKs`H3zRAP^ZzK>auRX?sX|3VDMM1MqgMR}Om-%<0 z#m}vmql(b^w9K2MSO!?yRWvcfb$9n;6*f>r=DoNyxi|pBgH9(ZKAjRP zxylHKb#%8-JU$DQk=gpK!@Of2MGK>TrtM_5@iL|A|LEhV!EI~5TeTBivMK*raMw@|)bIn#(oHv-MXiyGvyaYYy5-u{4ghGd=MZJ!c=Z|Emvh(Ysy;&gVu^Tego@Mo=VS%k9m-MG5m#c%k&xwCpP!APhbq@M0tY>`$^#qYS}=0-M{|;sXjaTTLlhQnp4<2-Y}tj=Ri}pfe$XuFbZT_5vbJC(NF_ zlTSZL9wyt!s4TgUODr0vhbbX5KJNVzJRP z%kl2t%(tO+>CldmIrL|M|AiOy^oJp?wkTacb1zMTy&Aa%bEijuuY!};`?d6qJdbkp zP4X|S;f`ASVV4*6EiUQVT>wGk;L+Q_<5XDG-s31+mUm^_OiE+rd&c#d&DOW2Va@%w za*0^TQ|D15_Tq+QLAh=p!2_sW3xd87T>XdQ(a#Gsc=5S@6JJX5U=B~NuXkt3%%*N) z*Jd3-DP$CBenCam?-PM!hDW`o$d)|mlC3u(3nPD2FGD0-Gp6u+YI>EdYld;I=aWKq z#_JZpfJ!pc`K~BIP6-oE^c_8ur9rp_ffQ`xQEY$#DHe>?UO%b#5 zR5~pCQEtGQE@>Q;Rz7^z=lLlV)pvIFm;r#0iV{vQ8gkFrg?&1;f|5!UB2NEP>#a^P)UwRfZRKvJ@ghP22LaQkWh$*m z#dOEyf|li1dr(*An=I@GWCmv!5q#^Q8XX1i$%r)b{v+bN^8^|;W@VxC3b zL5|tNohwd>Rso4c)TPwz7xsv z{2`w&_$!3}z?%8*e_;&)5S^+n?_w;|Eob5zYWY6k_UWN1r|FL521+y8_CT>Ji0#3F{~z**@@#V@X;c(n6{sIk>h|$v&k$ zc4obe9~{*>8uq2U6+warQ&CHCQOlsRj7k6Tz#n(&Hn=pKy_+hUyV?RkPQR{2DJwN} zQWPi;u;MZ1>ST#(`w|>VHIirGp^fIs6DPbYdCK+i+cI9RW&J7=kmh<mFCFRjW5QTFHnIQ08o(z|S%v~smg!)*0hMJQt;rW(Cd{k-L1mfW zk6t~zxtzrdgN;IW9|nY&+nb)l9^#VB(asQJG>ReKHU%Smgl&y}y>rVxcN>>8vjVbn zl{>oaD_Ig|CFGi-(SO4-+=m1yq?Hj!sZr=He8);nYkiBkjQadMeTALQn)tfi%`+sHiX_==1XDqH96%Dy~oPeclc>?j!g8 zg``Z7(Ju52U4q{s+Pay^zw2pl^e$;(WyXC3l$V)@PVf4gmtp-UFC!xD0EeSDf?+cl zJQx2R?zcy@CwupqoT~2Ot@cz)ZA&h6?D{V@DtcVz@W;4`(!FBCvAhq8$30#A@yA@S zf9f)aE^6_;Ze#ep>ym2bYpF=9VgPWO3an}9gq&hLGcOw1T4APi%ZH4ipCf4`u`@vD zm;T~4(k54%d(yTx02XwTHGbmpOdhk{4t!^=O<4TOEb@=Dt8>SduEh=Jpw1pozFupH zgP1MxY!c>Suco4s6_u&F`9V4p^_d*?b5LK#77GJv#U68uDXPVw$){uMO_w!ZoVb!D ze<^@I;xxB@yxV|t_LRZa?L;A#D?%X0dJ5hwhAva6wnTi9^T@$eM=hF4;)=Uj9;l^_ zDbbfIF0}DOQaPZJwa>(V$Hq!mv-Z!mi~)tg3c(_w?xcm5rgPnQEKL}_4oJWSz^HQ6 z30U(3^=LWyx4)FSzSa`Q)BQ8f)8;rZiot34WT)h&4`wD$uhwe@81_KU0zDVz)K#zy#WUuFq> zsk8_x&F=V$_WSKkLxb~4@$-?s3ANj7?xK9^sFDLkbv-+l9q84({WTbi6LQ}lZSCpu zcIOo9Q`)#cv%%sSkcd!V+#lu66Z2Uy1js@D!a3T=g<8=TY?+iQBp`QJto^dC>~gW- zKx$R%du=mC=#-k?j{W3yk-Lo)_LD<)Y(sQ{;ewd=twuS^53SfXxVHwa!y-ilSRP=j z{_V}TdpbOoQrXs`=)rWEE$8={ADXR3rjB55{`R-OsAgi6B+}Yem=BO*KwUxq}ZrqtrCw zlAUM$fVL~6G&Q3K(%O0?8v5&h%}qF*BmWzi36SEUp)Uc8LJ&-j7^7(VLhxPwW=RYn zuIji1EmZ~A2<1fJ|2Hr*0}9Oa+x{DvL7mZ6*re$wbuv4INNK-_ck_O-5y?0m7Ws>6 zEdQdKm)JFBPesLDxWA}|cl508UsRL5*MbT{HLCxh8ea|~2yct^uRWzjMe`x)L_634%Rnbdy*GsB?3 z%*WYdEdG^$+>7FK9`RulC3D4O;Q0^(8J|<80f23v$r+_{h~v$sw*9MK+X9(DJX)KO zw-2-%JN*O4HH?&3q+q9gqG21|U5C{G7j%M*dr9WV&6=)fmt&DRI~AL5LAGm|6f=A| zA5)Uyw*;VSP}RJcUf~ERFq3t`oI(D)KI#n94$O1y0asT#%U z9ix9$&C4MoNY!{^j%#Zt|AA3Y#^#M#<^GSVX?6I$PE5*nf;Rd$Fk?JWeE4r*hGi|- zz|{U}gyrz*yr#9#MxshaEl)ptJaV*Cqr|L1I~O`SB#^I5eWYV|C$0xLInyU-e8b); z2MWwM#hM=juk&+(O)Gr7Uc8{d%o@36GmTbk%7tUrTscdoIIEgDz{*ja8SjxuyCcms zK|tXtc$gLKnBam}*||f~8GC`e(*l*k-bXQMc8wgaYbddk5066Gglt3txDy-aRG)2n)3~o!w)`0>uc+?<)={!1f zdw<11LnsK3cVTkJ|K4?Act?t){Wyp62%mR!41BwP0z8&|uy@}PM$G^}lz~QlULUW_ ze_jW>6RS^z`ecV#Tm4^Nm}8pO`6$@26miTvIQ{DDek7+XUZ z4M(=gO1Fvv;ufZw=mH9MH~{%lp?uMkb0(?|En4hK*A@3SWu~J;Q(t3*fD|!MV8)G^ z+JFkzK+~z$!M7awUse;0qyb_zeb5>8e_4(FKdgq~KdgrRdN;}RcCK()J1t<{g=+yI zpbpurdY2cUI;b}(l^ zWwpo{0c!`Gk{y2z7C(FjI1%x$a~9w{^UgSuHd?T#fIG#Qoyc80C4-vSvNj@0JXJK( zENqGXkAbh2w1h?I;U6CeeaDY)eh`sWYvBcgI)ja78X^7+?raCAH%u@}S~RsX*qDy} zi3MH@?0TPC{)oKQb6S^^VsQlMNhT{{<$&rO=p94RM@IeeDXwMr@bo%t$jg@nGS~uo zje1Td^V-Jz?E`5|+2>;as2VmqDt|n8!s8&}_I*;B-lQ*{rGtq_%7FWk{Z*OyXDh;g z`Z69k&ch)BykZmRy$5#o2_%!CzDxx0thdkq2}pQc2vRkEv56xy-iHU;_~<3`@n4?1 z^C2rUwW;JFwvpy3jI{f^=(ek}N(#z8u8a4VgiEZ4{;C@JI_+=@awU_WPCkE@J+e&| z%9V;NoEjwEJxUVUb(@>F^l3!ktw@fe6`MWjD{s7qg1;tsn~iA4Vzd3>F%=F_OKR}R z{-t^-OuVR;h`Q*BR=;apRmlr^m!Gf1vVs&cfe%NS_?Z`kY7oZ%K{a1ZMR%TefFC5s z1dUywQY5r4}nOZE^J!&!v93z@> zE-yqd-A#e(hATz2BValsf^#GvcWT$v9UwLq-}UebppBuY{p#dA4nGj3?O1XF4Yo!+ zj{1vKo?Gp69t*&Js9|WOi0kJTCY#abM5Orujnj^WHTf8d)*!hKlS(u16`982r*Kng zKJ428#Dp>QA`#U0;E{vdd-`VttRdK7;tN4F(mkTOn3+uR6CKN0ky+`S2%@mt<#rUD zBAa*z$#K}vRJfAmG5j@rWMEnF&dA!rMeU6o>$bX=9ovK#?#m$ulVv~Qy2$>h7s6(n z-*m6<7UoM!-NhepIFVaIXAbMuVCD%`?`xr>xl3_iKawX?n1&c-+)7l*wmTDnqb3_J zS(vmqf&HZU*-QDMvJHQnnq2`m#^^ctJzny9H+wU2Va7FcIC_(}Z3u&e9C)P;41^YC zbV3|w46xN*@lt0y3l|QANOsVcawvI%Ow7g|babv#u9tIHi434=GCilQlj;tn-coWM zktKr^&nRG!es7tg`~%pvH7)%nBP}7+?0>z#gqpTk?pga|UhoH{gn`e|ryfet6(Sim zAZZmcqu2vMhj~@OI&l?8u+z{Uj-?yKi_ zv!#6uYMO{VdiHggn)1FO1KLa(neFK^TOou94>XHssieFAnwkcXskyzCv)vPwa1_EP zKMA=NoYzR90GS$EoZ!&Z_rIn_GWJHV?0_v^4jv&e3ZaG?5semPY9fUgHv%cVEB~#_ zP-ylY_^jnwqry*?s_Y;8Jcps4fFJ6i#qU zGf(O&=A77ux^L)^$_bt=B_l2xo?d0Equ?-py~2!ciL^WQMVGZ$%?^{45kdcJKAs-J z@AgGu*#S1>M4vDkO^e(-x{FLWcqn$tg`SN1Y&>eTNAj$g{cy!okg1t9V~+(Fr|dLzDb)NQV|O!bYTs1z)vix86zIs|qU8d~h4n3pZip zrD`QVD96@Wl^%Twoi+X+?A=vV-2H;?d7NOu9fG@SAh-l~cMa|k+#Q0uySuvu4X(l6 z9fCVd74P1=d-t4~)o0C`i$3eUsk^$Y^_O~npKqeE3WjHt=L7NjOt%F{-5jS5ly2w! z^B1{Y9Z#|BC66pZq5f6d1Gf*pEB+Cv%OIR;_CdnPPRlW0q(hosr_!LPFlV+d3m_&>w;<8MKY@{Lt22R< z5zJ;hoyQ=+5rSrGh$|bAEr2+(8JRUxwL(F;27eoM%ydiY=}IFU_5N5|jX~(eeZAri z+0SiAcJEXzz#MR0aM)}tA2oSE>~6{nBLd@!2~+#EN5dZlGjUiOW{rlOf*OH-(%wuU ztID|`w135H_V`oRZh=QxfU(9fJQ{z1e8525n7%+TMNWHl^cihu1iDPf55v}Gmm*!~ zIK3XfGO};Tc2b!n!}WK&2sse2P_vR5xV8!!>90cJxk(dnFppm`q6n##zIX zqmQ;*>XRYE9a1NZZn?z_sILD6qz}hCo2yNJ6m?K^xoPl)#lj4S9Sg{uCk~XLn;>UX zbqF^fiB&5_(<@z+3f!oKzn7m5T3v?_OR=m+5Xty`jVwT07tv@C`$})CE}c!~Q&9lW z$Z)WC8RHbhZrYrT%prz@P`BgYlmLsGNED*FqzEg8zZNwd|5(%rEXysQ`vPp42;`7B zI+LpktTJ>iq`}AK0N>tzJ;`C%+FFF{KX+WC(BQ~Obu`qE^_hbyGU(|H`4^uB4w%2n@ zP`mLe;Auo-13*GXauxkKdp;08|Ss@;@D)i&kWYRs9Fb)M{%T$vj1 zw5{QpQ6!GF!?olr_+u5~SRp=B>g1eSEbN7(c_gm5Qz1y^-{MKQ$2yV4a-@ndpX3rx zW0R`X@-0sq(_`WR7B%#$&mTl_yDPBXCM6cJCI*|qdO_P#(#-Tgc-elqx=@Qp9_>T= zOHAGys!ir+&4xdhF3Cu`>-1|DYz0d=TJ}#($;Pb3=Lp&glsFWn0CAaQTy!= zEMK}WKVi_jF4F?kWftT97d$hwhdEUPIqxC;^ z83{M$oxe6U*Xl@bRXsY8h#~4zJigf(330qQSw`pHI(VJM9dCyO+`HZ$$%y$lF^gw- zf?yhx934vXhVn}pAny1`n6aU1Y5dJ&4H73cFJl(dcXv_Gne!yNah=|im~ShMPSOuP z{5m|_s^YO*Cf$hLHk@*x61PqwkBE57*(Qi}-HTijs13m_TsX2s<}i|`iHCgdUhC(a z`q;KC{9RP-wjXF1@Tjd{3T*HovM*I^!*TJD%Ko&T68>ypIRY#I5&lpMrWHA8_q3hg zrAoQpT-mI2T1+o&U9Iz-@Kx=5pAwuB@MgRP-&-D1GOg$~ZyoRRcL7&5vcJ~5t_I9_ z3j&eq1LR0>+Vur!14=AXMGu;7G$M;lPprEU!wIQ#M8t>K@KWP}DD;rihA{TX!U#u9r`rm>*{dV%h7=-g z`x5VtrKN*&L%Cm9qSPO8idMeSdVYTQtTXbjSxuerW9~GbV9;pIu<40jtbL*qJ+Cwe z_~nI3a@oofs9j({&AM|2o$@eTA|(q+8Q};(BfLet+eN%|Q$7tL^?EVc=1$=I=dR{4 z0I;jsR0Gcf?rJ)LyPEBTIdH(P=D6mk%eQJ}mPTw5Rfckjv}ncdmDezTRdPOyzPKJr z6xKCX^EA$Z_@dPks3rpu;ZXY&NjD3Bfyx9dRjem4cr|}A%wFRjZT%j;3G~m4G=`9y zr`ThM@lRPC0cnQ~*mz$V$RH0(ud1N#c-rBOiCbP<^tDqE zKl#Noz(W-kxUA8oS@x5{ z_mmBlG!APQSMO&k`m`-b!f1KCzi(dcT?Z2uCORXDHV<6ZV2h=0OK}+UST=sSe|h(1 zQbO0Ii%Rt}etsmBQXh{aXA?dC!Bmphl3ltRQ!-vU&*f*hxF*3Ln=-3t$#Rx%XN6<{ z@MU0oF*r>C1u7=~0g3EjLO>;Gm7x``*lt@L1x+D@h4|$gsqa6RH91=2z-0{wU2ekT z63QJXp)+isRwO=l9y-}>T-mQaL?~bS?#YZqMHolTR!CEB1^ED@7;yjgh`*LK*;*b- z@T+d{dIEB(b!sPxD{yQ0`48>1&9F)0*o=jSHO1xSYw*2{;PZM~*ayT_m`qWG()`Tn z#^GJ^HDdBChz<>2W$@2K4g+;)B$5zHm)#%c6?FC z!!&*UI4w(doedb_HmUPO_UqY%Q)T~2?ZUw!R<&B#fV8Phpz?E*G#1E?;^Pty-Y%x9 zfkK!on=$%U&y^G39DzV{u71CxL`&PMK!NwJ>aM1H5~i+oXcjV?-0x>q&V zk5kztG+5I@D&;}q8yi80`*a{-o1!P8fK6*modm8084qFi1#UJ4X@$$=^B=#@Ci9H< z>Nsu$3g=vQ-ePIhXun13-lsKntbKtVNo&;>iRUj+@;1dYewm`Z7O4<^Ia?#v^opnSl0=2!uqr?8`x|XA_CU(LC%I&352!G#xjtbFbRAu!jcTV zG5vIialW>sXM9paq-xPID21%>a4Gbdx&sdVXO{=|3TlO@Wt8~VD*nyf1d|^8Zjs=R z%P0-n1xkW99Wi@wY?#vIsfGYwrns#Oa4WzWP~O-6jRo!ru&lAW8jod8fuqhTtg`$1 zu@P#7I*eGIGx2e8pt>4odO{ZH%S;@-t}`QE7r}(FBsz2rs@Ov^T)gq}Edj0*JT4dU zZ#Vk9ntv^8^1&u`1SS*!??FrlkLrG}Dut$1xy*OWfKVfLJlL3oJ=00_TnEL$NlDXS z%&a>8Zd&T0RAWczoi>Gim9v8pK!P2nbdNM;Hn4Le>0$>LGs4N@KwomnmdfHzuC~RX zaTXp<;moaDM%PRIU`<1QD1hLqR5`|oopa>7>rMFdgJHpC)X#u_(m%Co&@%@qfp zLDWjCdwWD~wdW6mN6F}l^$UYs3RmY69??W#ptar2{%FLNj8QB`((nY>xU6M0Y>x@! z{Ja+*^_%v>sGb3J~n8rWCr znaoISVG5sPhbELn&V5eO4}Y`$4I}oDnN~z3M|z>C>UOe_Qa}dT;#MMmfXn>Km%V7DcS2x>}HK(@`g? zWlZlp#XmmVv=1mWDe^><)I_~^TDCt9fpf<$tFIkpKcP3Q=VgkH44QU;Vug}V@t?=K z%h!nSmK|VT3dwGG_NX|Z9L5wx`xB}eDkbt_-Stqb2pqxbWD~PMc#e{+4Fi7?qxJd_kP@QQo$*tgu8F;OX%|c z(KGAA$b8&dMv@o0I3Hk~vVcEgGiJ8ztQhS*;zhqoM;5V+X%*Q%9;y7eYBE6$zUYi_ z?msJ{X1Grs(z3jkqWIZa7Jc7fraW0Qs7CKEV#bNqvtnkp3VnhdbU?DT7?4x)axKld zE-?0JA%PQwb%K~mYRp(|iHDbLu;nYEA~W0Q%lI*8VC2dHwg~EHgKI1fxl6t9b(E>u z6nV#(q{-E}b#BCY(O_6$)=dR(v61)c_c~8j7Wz0Kq@;tY?ia`2nre>IY;@c`XWgb{ zZ!=kbIDZlk6lQS#5@uT7g&8T604B5PuZb%^MDKo#orC{Vm??x(-lp3ZQ2t(gV`Bh! zavTm?+`S#fUK=EroD+_yXW*Deh)*gL{2g{R(0vhyxPUA(y$Z_!`|!14O7 z;Cm1r^W{FU_0X=LOEi*C3@+`UiE)>7H@RE2?;09>Pj)mpfM05u~i697gt1K@` zASymj7HF$SH6ZULlFAdye0HUO6zVDHHb^q5$gcoamB?gGunD50mesh;p7za~gYS8j zaw=2GrywFK)p@tTKNP++^K6$-qiPY{ubCi}nP_8Vi#PZ6I6ynVlyi%tqVHaAKXBE^ zsP3veM)Z4rXHegKUD@NVHnLZAV%^CoyQ^l>pSU^`i1dz(Wt5;NNT8lc&Zw(NR-an( zQ|1U@o%p@Pc!kY<5)0V$tCWCpJ6ag}A(6NX;o~C&M_4^wlh6KanTM)?`|a7JL|eMc zQSNlF!ASfaG6l2e(Ixk2R2|m@t~@=;PejlhTY7NI=*a>_BqFTbfgu@JiOz`rbD4bT zbJW*vZbPK?%-tzRv^q(a=jo83*__0!wk8vbSrHSP^?2Q#x}D^=L3Mg?bQ31r=u?M_ zQ#%Q@%^PV_@N^l22?NYil>U!dx>;{*})io?K7WYSYFdetrKU@z9ZpPAKL)h{3!K*YK zG3VHD5pMZLZWbO<`P2MbTJ1D(f(|!g7j1|Hp7;c`ENWw_wBCvQjN3*=jb|8heG;6T*PszE~LYfp5;3%8KZij805p| zy|2LFi5`7wCe%|gX$g>2gJ2Ieo#=k2R1wgsMv}!amk7^o-sFBl=NOy})4M#Yl;r6l zLg(F$9uNtg$UuHj&}~9drw*9(W6aP{cPeQ=GoStJ$S{)6R_`06=ozP9{tL%vSxpk| zWUo5xW?N!JQ-Poj%yPFu46YpL%XG5=d>KItb}g`S>1wnr#!X5gi;BBt%KiTJT!Tw| zrx=7X*wLq(hIuF>aVBJ)h4is4EJw!Oly3S40)!TEhhoPdi|nsm>sH~G?%~BmrF0!P z=st*eT{Sk0LAsq3#JZjE8foa)wE;~Y0{LB(hB8MlMv8S!0AI$KwUQI5jJ@cg;39dI zB=hqtnUoISatzz?mxOgfx&E}>i(?ZmlYPGDec3E8ZtwY$aGRW4jSFxGH>Tp}#XSAw zFgtIKw@Gz~+5>Qu5{k-zdFd&|F&lECl6VUGtcS>C6l=ODubdt;o#sYR3$=#wB`HS* zXzu|)V2J>;!Z8QG&kqn-I+d|TrJwEDR+Zn=7E0stZyJ>{lz}J5lr=@GI*&fw8Yi5) zhCz@H=j%ADB|fnK5SJ-cj?aV(tQLqUE?!v&8q)g~mJG0>OkXa?&cR}%pIw_3xbpnmPZ zv>feE-8qZ*S70d>oI0ytJQeN4L4ON!lwdlAPaX{!J0AE zes#L&joPOu0&JYZ^GM_K5g46;zKr3!FB1^EfE|jDDm6OK|)BtW<7zdxuj~ioA$*O{6-GX4{ zzS2ytvIY6ine)`-u$9Y{t=bqC9mWs#Z4XTbgkjuD`ut_!swlHO`hduiW%jK4+|gtP zo(+oomziYrK^$dPUoFY7VyofRf%;-B>F_l>)3n!`TFP)Qx-g$c(5PH6BSWwL*Us6= zMnHRhGof9qz4CK0XJg`6b>)uE1$E`eW_y^_<+N_4sO?uxP@>@qs+qUw($?xyi$8}~ z41ZdZ-Z0g-WB+3Y^Xu)?^`q?Tdhq?q*qg868~6EqZ9|5WwJ`@a+z0pE+IgSH6ZFlq>zlGpw5HoR;|voZ6Y-r!??POrToj5 z@%x7@Q?778z-C)<6wwtmk3CgOYCdLJ?=`AX$l|%mU@#`3c>D|V_frLucD2;Qy%3WX z9U!taeN5_T1f{KibbE`$MKW(p)+1wlU&pF=_cas4EP4R^auY%*kWqCqCAGSOrpbOh zA!*h6vLor4f;0MLnfe2L+}k6=(K(;a2|=ezEMV>E&~-;J%lduHgfN2DQN^x0pNCJ( zAyF{ErfJ+nNpJ{#VJ!B0co*oh*$~#^u=ns%#?HKMQj`g)YyDu3x2!0k!pC6=s-Igu zh(^TBLnotv&XNb4yu%b4ptCe>?q?~mGLRKPIWxtgo%j7uYBPPwE)8nmRU*)qF~h?K z*fQ2!-&Q|$rpgt%)yTD4*KS}7(`ChW8agSh&7ivUlv`LdjbA*{P+HHFbCH|4(Dm6T zouD0dU9pOJV4Q& z`fDXpR#$DZKG>P>T*{X-UqA@3JMedaL$+9QQhM;ZK8O+ricUGWn~s za9Kdy6qTDO@WZZIDB*qW)O z`^e10i|p$cUt_6a`A%a@P8|A}jOy^}z@|Pvef{iPpiD?*1E)=YweyUSC>F34F;pNs zPDt@3Ml?D-;j5Xq+*h-lnUIJXt)?;+^(+13#LCkcg&7{tPa6&luUQvTtDtuqZyk9$ zac=~?z~<(a=ASpm=SR&#u({!hmvc%A>)G>lE0<@c7us-+LV2kbL{xGzU2QO0DeT}O z@@pK6-p@fqzf5cz(3M#X`CUUXlcpHMMm z^F{!;OsWGd*IO_&0GD~`#e|mKepdmaD@0TBOZiezjlT<%RO~IMMpHzajZ{gr=`J$%s zdn&&yD9R8P+MZ$Oc2~>&clmryhsHsecxfVUo)$v=c!ytI;Xmk~F}t0gr}!@EpH)4X zT26kv74tnfPSZi-O!3=te;9Of^7M-1Ysk2%4_`t`<<8}2?!iO-QX?XFl&UZv` zGC%Hp-v2uD@&v1o-PjUgqBv|ycBgXs)qXF7R+ibTNn%^BCYX1plpgPQJwpdQ&2 zCa2-*NN5sF(7bYbl_o(c_+D%JlpmYE|11>iRFwcZ?JyC)O{~cG88@CzFeY&(tv}<}t)rm_m?Z<1dicNSSe@ z`YPjZ23qJ7D`@ndBCql$m0+ah9;Cp6GU`_#RK=C#ls{-);}_Ybujv7C8K)8GNKr2d z6Hy9OluImwJdm#WWBNv4a!>kxChpcYG z#Lb_|$j(m^?*t0_lwM#PETC*?|UiPaEk6%Bh z8Q55YlngqOp0S1ey##mByOf{fr}P`b9$#Ki`8#Y3mE;ZzOQJudkjiw%G)=kb5Qkb@ zFtz%%?T~$csJKbs#RjbIxP2|^!)TJx>QHscv(bWGIwI8O4}Z!B04$U8Q8H1KZ3Z{Y zkVbqz&u;P?%)P;y_8>au?Ew-M5SD59>P1VlRYd+;_4N9>`R0%w=C>Dk&nbb!cODaE zVkQ;0@NbT?cw78leF>N_D?c@wdEMrcPniqa#bQGY+dZmrK~)E1=iVvz@ckUAY{uS+E^#zQ!Q z=f3yblXtq!;Mv(zy`KN7_CD@qV=njbQk@`g=CM9w2mGg6++mnqZUo|Nsdk#!7CGS& zHG+|GVHoTg3tt(#U$8+)EPil(o6uo%$6SuZzrJOTv)=9<(#}gXL`Wp4z;A@SK=ot0 z_R1Bd(_6W1*u;nK9?taZ?D40^kFvj={+6yyBnCYfOr;&&~#np(3R5@|ij{ z5GYrSEQ-N?NfUlB#LnC!IX~#beO1Z-kS7aPTQ!$Ell}}P{k5rM1>+uo?z?^gS;$(M zf8RGmM)QHexI3W{7v!5esZcWD*t=YWM-OWg@-GzKs5KC0#X%oC8k$6yygtvgf>1&} zJ%ljR7O|oWIenb_RIRS01!*+LC$ZG5nID^rd3ug`)&^P z%j%es3|WA}QqY2GD6wr~|CPCam>)UOm{8<=UuZhEDTYmhf(KX)u&}fMC@ihK7nT5n zSlkNw&0)t6{BmB(X~4n~NFK;j;(K99)4q_LuxSSYSXlCRu$PqKn=!*+u|Hlxk@fgT zVF|vWkBPJiqS8wHSb39Yt2B&+i2k*$h=H!FA2IHMc-F+RU6=z}sc8ysP&2t>s0 z1~m)G#vFbV(^QV_Ylg|!Lc-Cw6$SLtN z>^zYJ(T|3Mgv}#x4U5;qR>^KqZ<1z*d7mCSI7Jb?l(lf z=2xD6@aWFLWDGcoa+2J{QFg0KN2BQ#27#DTjAG1@hXo57q{u53%!_y%AF!phmjq*Q1XGN@?!IO5?DRqUsDu{OG*e)}b83R*SVdhzz? zw(|_>T{o;BO|LeWR|~InJnFCX>%_L9?<;-`feBU< zGgWRjCUWaI1Is8FJdrz6oE{(eTn}JnNV&y@hW^3Ih^Y;^99cv2=t=Ag)-wVEOW`K{ z@bMl>&Szbs?6f`*KIUY~p$U56zh~nMHVY4Muy*icndV)Ov|-<@{TSaki+ePD?k^@C z&d~w?DJN_!cS8*!0aTgozf_r%oCjcENt<*K1I7YD;=QlrQndi=EAfvM09Ba;atwze zFI9#;7}!_3>=A6OyJuJ7Afk>u+}g_UT9x*LfvZf6ZjllSJ~BBJ zh(+CM@BC~7?|Mru}6H=Qsv>IUw#-&$i#rF4=8B?>+z#w_T5ZOj{@;VhT~eqt9Plxa*KYcc?$GK9xAr~p*PpYAVI27}!Q zkXKrf@K*kn9{|iNd8C9BTI-%>eIxogK^g|gD{;a%$O7_8!=l=NywVB@!_>d!m1_Qx zSNa9t^zHs)ALkN~SCXQM1?Bx?-TSe@A(R)yKVSG*{V!C;jN^+W5S6h572J;}zeZVX z)%%`T@?^(ndPimWzy0LS@rV*my#$~#3|MuVfV>jEexT57U+IvZ+vx}pm9hU9 zDq}BD0V*!>o>y`SY^>5xBeA|zw?8lVH&g}(h{}kWM5_(GS_-ALXxVFA(RY!g5(=&P8|nS6_*G zwgz1ksLCh?!FEFyjAt104wK*1-qost?pnKtf&k_vuNo;TNgP5o@jHw%!5Cy3tN@!Bs?H9Y2i>36@HqVTOMO+k%%t<~ z6y}K7XlG9NY0z(y1Sjb-hA$H7Pdm%3q6X2AAqce{=97|mc_)ZXD26}Wq3gIYcYcRg zuk7vM1PeC~aR|eSF)&wE)yE_C_1pFDEoCqkq%`mMl_Qab^qwk*A63g_Q>rDlSi(og zgdwWu^61vl(ryD;8MnUzOV_}_(nxGK&9f~Om@n3sLToIL-C9OWeG}Q9sF#0ZWrVjl z+6ZzjL7>$?SPGVZ?Oa6D>cYsKJw;Dj&*gYV7=8~dp`N6@2bQYe153`HTO63-3CLAO zjCW85IwRd*n$Zt_1`mRM@TEK)iVl0`6a4cuV6{->y#k(%qRoYk+Z;Z?mfQhMs8pQw#hzEUGwYq zbCvk|I^U(cPLzlArJ>WnfQ+^NIra;`h8P@D(vbNTNH0cD>kRvcwv{~7jyE_2)w=CpHNv+C9T<{8)de2KO3{`n5^X#${$K0Qrjt-Wy% znn;RcsDJm*D z@g!QMb91NLv2+gEova{%=ytuUZ+mSps&m`6tEGAVxYqWXv(1{z!i1-Kxj>w_sL7I9 zauTBuK|7<2>X$BQaG+&egNHO6Tix@)3#n`niqx7h##T3d<)wgh*u7g_ymsaXb&^apk(lg@Liy!Urb6yh8MbD07GjC`5v^%W zTzLD6zUH0~7N8G}4sl09Yh zEsIV%?B~?W+~6*bTg^;4p-OK|Ro%#x5e(?d@Z>+Hg|IyA z0DKwdFvH?RzkM+YVNFRxf&nuxp(~{Pz-eG*sf!176;N6FNYnOLWvK^HSsHq;EPV!4 zmgsidQbR(J8Sga~ft4k92`y&L9cf@?=?3e)vc$)y>i?1^t)|7Ds*~INDgQ}flz$q} zq=)HPzizhs9J5?P=ypWv+4_qzJfN}!+M)=kES&%=OH^4}Jmg-$%2Lo-{d;AJj{gc! zS)vWY!`iV@V(>A)v4-5;$}>2aGX#P2n@;aHXxh>Fb4EC}Z`7^aEH&)Igdg_R$mqrx zD+s$8YI~R;)3qbS6~YWGc9YfcIpW9HAE~a%WjC5e8?u=!nFUzt#M!ZkBF&j7<6)_a zLDq*|BOFxV#1>-I!g>e9>*KhR5uK4Zo0 zV1*xl{wNiz_)<6}?g9j6_^+I!y{L1bXKPcN$Byi1q1+9nKm+HWXD5+b%zL32TpTK- zkbi43!-07|F9LxX>Hh@G6!ckHstGVa@3ijUy@1O$E;D$m1{w{DHq9+U3Ij7s9zI># z+*hn9IPaOIT|j0j4?8RB=PyD5kUOdMDcCRtirjp_+tvE7ikN z?%npjE(N%>dTMCot35xtQ{Q_t9qYe<83>lxVLc|r7Q78=ATXm|JRY#I#@XhJ+0NM( z)IBjh`4sK89{L2#EM>E&q9Y~HX{Hwt(eJcf2Wi)NoPQ2+6UxB224t3INRi;tb)8rq z7y+531-}d&c{lkUxbAu)%#eZXs(Nx*!$ataVgN8>4*+IxEdanw0+R#v2ifB1Au<3k z6G?0BnhYO6V)v#mLEh~b9?w{yuQmwCEHP)&I3pLW2T zTu&@Mr7l1zJ&$13Ep9Pop2E{iKo(qmF*-MzAo@nON0z3(fGL6>n&zAsKcpoG!{!-5 zV_(gtrlO8EG+gt0f;}j3HrmSz7UF@&Tjl00aWRWeS}naM{po9CE{#j*+ug&I7tibV zhO5on(AyJI7G1*rfbSQ41yIwfY!fOlWog!LH|j^$fXtF=`|+9?NJ8-1b5q^dR}32n*5}h))HP~)qEa&pCZg^>vtW45o|!)*HMoB^@7eXXa@^7|3@6gh z_{Jcgv*Kouj!aXm+prYR#0T=8S&H$TcYGvr5Qj5QE+NV^d(SM%zh{=(=f0U=i8pRb zAa3Gj-kF@bu*5(ydZ9STs(ZoE_oQ)CC&RdKcEYcyeHYY#j<;Zew%%A2uB^z6g^u*W zQ8c~(Y{X*OT&^6*_`S&38a}TfLhvX$>(C(G>{3RB9tFV&4WQ#Nn-J-CCZ8!=ynf8{ zDw?K)$|v1ZCRB)oZ)#NMG9rscEX_|Chx|#*sH-;ZlkYl4n?u2$GCioG&}nH)bYsw) zo`b_)bokveNg$7;kqN77c}?XsYsob)>${F;|4UW#M3sNHYK{e+lrk0bQVE+`JeWUJ zpp|33gcRGjzgLe6EDKHiY!xpZfp;Uvd1S+ z**^}uuymJFC%c`H`;58%M`lTZf%Ui~uLIntC5smzAz<%k@wQ|n1W^rxb<*4Io@eFS zvJr-lww@ggni|})<*1W8{kqh2Sh`(B+gY#8l6007-@g5tMp9L0nBaA{SU1dntKIvg z*MK~?&;o04*66(Vdu&y)zh!}yW-n2nR_!8m*>o=!UsCt@`;A3^owUzIEWJ6@N`-Ew zIi+U_w(i-WU$h^-z;%I@i8DP}uL9kQtht0Z4u@S)ZiAZor$~M$pN-`pbp?SISSMr$ zczO+pmc>D8bkW9&3GN%`iG7p6y1BWIdEvy3{D{i;%#s@*vqYb? zg3D_0$i_vcl1iUD;3m|RY+_+8$x^h0GlQ)!mHIt~tZ8ei!{F+PUrn-DL<=3Sp+sQT zWyDnu9pC&VZ2^qyscGU)wgb8>;V0%>e8Y8G?llS8qW1cmDxBEnryXsLYsT+GPv^aR zPv?0rYkp~trpBk(Te9!xI^-dECUNE`WKt{wL65ZtO)c2_+dO(?AN53elOh#jT z-$3$1%u;24K=zx%J(#X6|F!hNK{A#SxhRbr^Mz=0TCZlLPGx$Igf;9nwx})~aqyQF zZD#$81KWy_2SKUaywsejpU6Bd!38M9jMqx0oys9hC8EbUKo6~>hnpnTGT3bKZ-G$B z%n$ux|8Vh~p;lCOS>8IZ#oL-xZtHFB@M(d$g^CO3a(vc(A-Q_r^^WitpQtz%SyGfl zg4yW!qS|P=sZ}>Mo?xnpxmEH5{3j=IK;X6PH%Mk{1Cu~VOh-;+WKi!wqo-S(`NoQc z{~Y8|rzSPVtUaCzmj@jda;x7cyuK&5yyISZL>;8cxTwHC8cVe=Y%qBPU*}pdM-hOH zrL_0PQtm$+OJ9C07GQgs3BNa%R+sDI%=7A+Y=#*C52T~sJG4o@KkAYZ25(#LZ#hzI z=IaPqy}I@ti$1abc2XU0$#*!F4*KR8kp$CKIrpNX?LjskWxgYpP45gZB7`IP(Azh1 z$D#vb-FTrHGNml*#tnEd1U)<~;#Pcg<&XS`i1-m-Z~*T-837@Q1?M%o@9K&*u!t7x zQuz!RS@L)NhO5hAWy!az?qKT_RWKu@r(a7D&uMYXzh4bTG)Wx%K(hkteo08D$Qi*vclq{m)3wX#o4EUS zc;n9Ya&G%pd|`5&->34X)G-k$!C4a}3b?lof8 za~I-8OWNCfdoLp#w&_)vP6n^=z0EfA6ElIqO~x|;-~IgMkGC2>ozI<^U9N)`;9Jtd z?HZGRxIVZv@HZTqc9ShXxWi_8cvzO>eKGTLOk6}XRy{XKeEn&3xgLgj^XMJi@jrM26Z}l^5OvbRkEBKZcBA`y-5U?TgX8ZF z%s)0Tb)C=9?J*dmagb{rgov1$ph<(?FKhDKFXv(YMZmt&Vrg`7;QVm>D1Z~Ua8s;0rWl5S-)VtRQ)?u5Lt+a#%&XR|17!z_|W zs(-sihY}EVPZ5JDSA(PD9))+?tq%a zRl8w06;ZS4yH!+t>e=BHNS6B_Kw#9O4IsQ%R^V`7?ps$p*?)O%;J-^P3UajJF z<}J!;SLXF*-MbxiXrkZ86k)LCmCQ)>aD<|#Kg=P;o-?n8+sOpa$?INLQJgweiZW}0 zX;QmYQ>jd)ZkbpFYM8SzZk=F9e}89q{#Lrbo>lFFqfKr$K9^~@_`t_qy#ID`ey79z zep)bwMFWlY*RWG_x2Y1=ST0s+tubqJZc_MDWckWS&EO{rkDjgMkgEu%49b=j63#F) zWNnrYvFx2i;s=&70on%Qs~PaIlr>0(d2cqqezi`Uf$>0!ks8V+M?FNo?QP9`joBMr zB&LvK4}8Aa@e7b2$A4p{ZvCSRv3?=?HrVz@_ojcRmgY6@b!T3@3C~m*yc-x;TD*CX z-aC0Daoat4>{oi`8q1S3x$8=&yHrgTJq(X)vXN=oHgSD)#z2cC2n=_sam{>Py{K}H z4k~{-2zC#gDE&_qOz*!a7^hcvXz2f+BpBkqC78b@nE%NV%y9nS63pKc%)d!6mH%TU zn1lZ_B^X@*;PRI1oO89^Z!wTVQBtk$AK>4nmNEU(gl4%d!EUEJr3(i zFl(~-l!Y1b%m@ASQuH{1c7TMJ1G{ zvC9>(fO(BszjE6F&@hOPTUpCsgJmYq)Zz-ywD&~!bkNlNLu=z=Z@z}O*C+by z9PXlKxwnkYqT5u$bowx3%r7Pr=TAHowBXiI2cH8mKkDxW3iVfXT}SwXD6PRp+WCVU zpzAc@Lo}a;e*pki;O%jf_lH_ZK7cQcWt`i-Mp(G8;b?liJ_Wz*-#k|V++G5ne~BHHWRrcE2dHdH}kCejhm<~?J}U>#0=;+Sq)^cR3ZpkB1=i=FA$rA2Winm z4XVLhBatGL6+C@)24%D6mh=;ZwG+wD3MZ`=c{G};HRYpyz5}FmwhcW#Eb2#& zW)vvx2MGs^u+>=i?lynctRzU)O=c?a|L8XvZ{UQ}fNSsE5bq)9!-xgVgkOB7#-K9a z)32PfN$}-!9d}7I_!jCT!Hc!&Cc(jAE`!}#@J7F#4`2NXf-DwEyj6u49N-k^#Tl zF}~rz@b6$M^y#s9H8%~>YS8z`Y;RF1{BT5<0_n@^@jc|@FL^}+1e|i!qLD#UXF)1E zTwImOFwmBx2hX<>K))%etE`84$6xdK1dP~^H4TpU=Q?esD-KLVSfCDGgnjBn$5i-) z=!Ri5p^tmvxoP0n5@5e616gB#;gUm3;x-0b-X;k9@Y}9HroGOVZb_iDMuLBctAjxC zE&nqs!)GbCY?^K(KWuTM>f{owkGAM?D1C>tN+yD~z^WJZQ5b94_mEr~Vn;P|sMB;Vg{|GW# z4MOHJ9ne&S_NYB}vzX6?`>o`{pARfi3w$#gO^5A4rS`V>P1Fe@3Df*U4!$Y2;+-5T zUQI|sCS`_&!K_W+8NUBx70_?Wua=@vk^iz%4rS-coBiN(jR$?*L}uP5Q4(puFhTUIv*FGp3TDwC?r~JWGT`WsI-2_Ov&3 z5&HUU`B72t$PC?cml5@WEGmWx?Y>tG-8(Y#3GZzFs*9J)WAx39@Bd|7^BFL%G5^=N z1_v;%85`pKZ;Wda|I4@rwLoH#dv_FXYVj17Q5vENJ?=U?h?{!eI5}MI=I13}v}Zr2 zCAYBv6X98#l9Bm8)-@Dev(b$E|Ge~~hFP5|BVHyuX|6QeF zHJnTUB8l##wlLUv{N*LOB_EP)^f!`h{c@{9RuAQ7wgm%%ju~^q1L?9|LGT}hiKjTH z_7pL2EWXf|BHAtx%uUBn=%Wzrc)C#5v|orSz=s=Bz(4=7B%W819Re5SkDy|GPOHUc z*P1I8#A;P!`5ZL0BtqmkDC@{H&gx1-gBcpc3=1zwl2RAVpeuD(%H|y`-=%nnft4KC z*A6Q@9J4vcZc$VYajBC~KivYN2C)Yxi4)CQl|pt6L8*wdC^4LbK%cK;narZpyT_Ul zdCF}1y^kGg+j)7p^hJ@&<~nY?%%=Gb;qF11@t*v9MPLc_R}roT@(2aJ$?WQiNTtw{ z0oO^{1qprQz}m;zqTOwq<|27r>+f+thar$e#J{{am>AWHW|u1acZ0lGf8|i_mzBj? z;i-RJqqwfIGiT7XjWS-uaok$MQNi+)O~pj0*K~1HU9eQgzvZn~rh{&St;K0@wTURx zXpwok(We_;TW z`aPLU<|v1ot#ryS>!q7v*qFf0YpZiLXT&5DRvxa?Jhi3NpH0h&nxa|@BWeW+MveN; z*m*ahzTa8nGnulPLWc3$dzvX`bK$+16Q@KVDt7lRzKY6sBIWwXuEgR=44cjUiI1(Z zDy2xvJP!T6uoi&Aq9LP;Z9}f645=8k3Q^5jCE1O7f}*-t;;4EQXw2hS?+n6tH$#CF z#JT-EL(x<>&G=RQ=R8^E^NS){B~(C+`n)ziw)AKxG3!X*nVOe{%yMU%-?Qee!=W{m z*-EFW!;Yg<`Sdxxy9)MTr-chaIeQNdm#V<%>Fs!WYm0j+2WFaa{|a+BBn__}?wD4s z;h{xFrB{VZBMhU?^w%+;wwn<}gaQV(?Gb!+B!y~{kk0zklkvcBYNdfs;?h*&;1p*{ z^%yiGy~u+fkvx_eC8;S@yXg%gyT67Je#UJXYny4%=^VjO{8&0yw>8*!lT!40`|wm|nL@&j7 zFCXGI=gYTQ1I1=&s$`t^)RNcjv-;TEUA^<k**5rfu@x$u2gNIxMHTI7*Tw~u zn#^$1@30>uUy|feK8DEa&85`(1=VMR2P1jhx7Q9uA=R((Cpj7M%!wr9#tMgWtnTn% z2M0*{_tQ^kh-*eo(mpz5jx+NT5;WGhOyB69V@4UR4(>mUna@SP zW<`o@bdrF^Ovu~lzl<4{i&Oc38Z*ECSH_HH#Z$z)F*En?#tgy#voXW#etVrFG?=`N zUzf9KFVdS8>*AGI_$MAxsB_r!7v8u%W0d7KTv5bn(F$CYB})pMo2F;Zv_=@iMgaw! zceHMa$y|MMy5OPxLdP`jLuAclv8YC2)4-h=HA3IR!!k1!t7;UQx69k>Zxn3(Kuvww zPx(gFMHdBMU)PIY=VMV#)nBu8x-#F6AG$XUU(VehxWivX>%!p{TMJZKuxqeD7{s^P zI?XO(`AU-z_T(2f$jmPuZFGEygtKUFIVG?BcEPuX<%m2-Dg(b<-T3bg*N`fZCk%dR z*ZG|X29528EVZ`gR1(aQ-&2G)y+C=ZQA9ce-b`XJ>Yw-E-b^mJjuvD(ajn z?*D!LDAcNN+s&>h*E{4a&ZFE7(r%E-sIxRAt#mw2>C^Nxr3hQ|_`gE|1GF$UzaP!Z zyNED%r?5A1b`1k4JMso&o~))?(6xlWLC}|hbCPgM(nWt@w7_>!9&FAcXNrOuB2jS_ z-fuwhCVkS+@C0?8Y-dNc@!Mxz6bfPpzxL@gBvitmJu=qG^8(T!!8x9F?6emYVoSDy zlBJaP2{bA1GQ{cW2yI%~c0JhNSss2Wwpuu6SUXlAuUAd1B$a$yi9-`y#>%2B3_y)b zMCv}7vxZ9D510uDUJS>IzydOg1_UhGk*QQ%jLm!k@j0?9$J7$^;fUSIms?|1Ad;Z= zMjeN#EzgDgfCs0er5;j3pM0%3L!|tbK0Hk)8)0#ZeSjq=;uvDrP4E5)yuMn#_~o_T zrGcfi2A?q45y65qKvy$!zx#k<<$J=l(DQ9qb037Hp=_-Ls z*}LKRPb5Feo4nf{zmu}`7xZ$RwM)?94gMAV?Ok=)7x#S| zIBAMXj5es=t3sd3tebfu>`V5+$*J|eJ?e$E%V`^n zk-SnT#&XdBn=~Qb_9z1WhPA*uU!clsd+v1bh(FzPE$Y{1p0v(D=}t=i>_&(m##z*p zvrBPVwz==YMO;sVwM-8p8U_ZMdX{YOt{u1`YIH<5h*T}CJvO%;V^^-DNesxZ?PEtc zDls0V5cMm)ch7g+B^v>ex@`E>#0_aj@fE7mpF zyfOxvB3c=3odN?>BTYJyb;-D|fXR8k#8L^oqKkkIp&@0`2~4*ND!b#c>Kxa7^X6<{ zE^?rZCIxgke9-t__lM5cr@0T^ciO-gLqQK8po)Unk}uS(w3;O*wA{*7FuNtK`2G81 zUrEXo{lImA%BXZ)1u-cu{$yXP3OEy zdVKqHA%t}yI&{`O8FHG$<{{;pu%Yns6H!09g@InXdz{v$uyeM>Itx!fJ!x9IS*bKI zxK!rJh8OB~qS4Qrp>_ZVlcxZZTrO$TPBP}+fPc;glR_PILt`%jQ}kvx0=r6;)+O}* zIdu>LJ)k5fzk&iF?NV_u$1-AWzP)>KwUx^^>fb|U-OI0YybgakV*;s92E6eJcYml| z67rk?x-sQ4c*z)Gm{D86E=SH?yQHCdB>K!y)@*CJ4&H3HeMCA6jA?;f*+nla)G4nG z6cV0Qd;V&l{M@FIIW2+S#S)=jPrtw(r?@$?8j89ErDEPGJ9~+bn8N$!8DtiNINY88BQW+}QtNLE)V>ru5jVX!tCETQA?fSrGFmZX2@EQ!21OD*6^7*OkH66*368Q@kOP#Ir zGxwJyH$fa|g|q;sgY{5-XXALJ#QKnPZauxd@AYIKY<7SZ=jTFCT*Ev}R)!I|g+&)v zL=R3`VdCZNd+LHrG=UWsX|Qtv9>MIe+L}LWqV9fJSRa-P@t1qLD_i5Sq$T7i%+X_! zMZ%=uI1zm8x*+@>flfDHrlWt7xNcBRF^R=s?|~ha0JbvBvx{!O5oPmC#FDSHN0jND zRu~%NH>IsivS0=vE`7l8N`U4_rGRiZgaCMctEfQg<)@*+m~a=%ua^{!+>3|d`bdKC)y!~N7XwR7V;Khf)908on30--tq@W@}`cTrd+DkSIg9` z2SMV#IV$HQav_95m51Ra5xyTp3shn0O`&4L8+Jt-h@ldFHxqpV!y5$$*d=5u?g5fO zC0mJ$;_e?&%8{@jv$46;cs>9!NL+>+2r3Tya6HYy9D`^fR>Q`rFD5 zW!IW@vS2R37waO@qA{wcd?GNtKz6ie<3c-_#VG0r%233G@-JHWUu0iJAz8rkrJj&6 zr9FpjcN;8F+ZJ&fQQTB5O{DTWIL`V8%kYX;RapBXV24m`$d!&2VR}k}a>O)eXlm?Y zZ&%WMYPh7@Ke}(*i zP0h8p$M(^Jg`8^&@nRr+1bA2BFA|)e#vQg7*3(pN+ew${K0wk7D%%X8;ChSM$SB4b zbGI(gXjAdg4hp9%v)ra=Q5g0n*U<9r7NZ0eOem)ny=oqyJvscWW|{LSrXVL-@qc{Y z|2t-h^fzXS;#X$L_8&1zAOFZKW&DG24eWn`S^D|s%u>N$nWg66m?h+H<6oJj_yzOh z5i)mPH@C=Aax(Qt;x+A0!u#bkKp!z~N&UK=H8Jy3KolL1683~Ll#GM-UAmAh^`fY)Sn;F+3($-rG`Sg=Uv%`vcUWmBMbV=5bJ{5o zQzOXzx#Yn~nBk$&q(DzNvl<;oe(ulpr~fC+QttHc$2D~S8MDOm8?!|E&ofKOe_)oh z{)Ab|4MOm({R^|i_CLiet^SvorS?B#mZE=Uma>0imJohrmbkCiy#9G+N$zi%C7XYa zS>hG)ul$W!8vlR6ERFmZn59~Y0_o{C~pe5uHUI8p9kQl8`?6CH;Sj6M zA((;Dw1x-Gn1v`p7_%4mzMgH9I!bd$on@$*z7ji*fIz*Y+8-GyGfO9CNleM81E-ZQ z$pDD^Of%;dFv{@!+emdQv4^tp1B-BeTZ`Boa(2u6jO{Q4 zA>S&}S{<&5Ml#$j@N>=+Wj^dfaAs-Q#!r-_{YmDsMsWKiljXV-~Oi( zGvEK05;F$BPs|kD6}%;8-v3L9ne^WhGdzEim?8d?#LUxwmY9M2`^3!X|9E2Nyzq~S z8O(x(Ny_ZaJnm=0e@o09|4m}%`+t_0c^l?G3-RBb6NwEd>IDr`{vs@W5~|z>Zk!98 z3oZJ0rT>9gQvN$)DfJg(X`TL!SVDezBbGLkx`2&LudVi{S(l)J9o|E=Nfd+ALBv6u zE_q-nC#Nyj#N1y*rim-_&*guKRqovIN=KOD;yEa|>Shp+!KB#$aqi7m( ztFJxEZF0j1?^kc(aB-7pZo99F1Wf*gSRxV)E_Kxadn1;r2)qWb9Q8Yo=YAoU{1=7F z-iW2UT!91smh|bgIJAC-F#4_!NY~>+KYLzWu6Oku-;7UP7!f%bj)GoWtO3!CwgRt^ zxTg|I1Tu=7C+AJ)u_Mynf?4d484g+aU7pA&@%bpEeFz5R(DH$li}>2Pt)zW0Zp(2h zJ*Jr9VQs1*D{hl%r2})5v zMsBwx4P$+t*vPt&H;+%TG0UvULfO+t6SHKE-VbPT5U};Au5EA!u7o%2+X*B05q0 zJqdI#u@*$VyXjzl`)UnH>?&gqPat)zV#VXjD9WKsjaEXR3h=#$aZT0ZBdJMXzJ01` zD3uVK2M5 z;|?e|A3016Qz^R+R+D;bX_>~1Wa+C5(O|lKz3oK1M)Meoj5)ad06 zBoxl5&!JYZiCe@pRh4oT(HJLJ`>0Xt0Wn5Hd(YaryOgT$4OV5{-%lY z%vIO{GoqD%k5!Uhc8ppJIg%Iw%pfRVkD(Op#}_yFnGZz;Fymy~H z_BX`R`kxR>e)~NXJPTT}ihZ`Vc!Q8x-NDytumWXi3j_9=@un=_@J-=pIu2^X!bUXk zIX!!`ZOxQg3z1`blV%}4ypf!IshU~;isQk2fm+M`>{^6mW z4j?Aj(lo_LLMGkNx&{6`xszF%IMbqiIQW&O^kEHu^4=Fh5DFHo#qu#uQ#*whl``}j zZoYdQLwtX+8F41+jYUSHq9)NE?RU!8RNZey8axphT-YHt$OB4OF8xK&ZD{&b8Hwsp z(V1iWAX=c5A~{o{?o<$WQ_CD3Ta(L}2p_V-HlF66MAnFBsQ{2oU#kMj?3Q=f5uHhe zik42I_I->S{dB0RMibW@nkXmX&%dO`wmLHBu5Z~lW<`k`cnPP?gn(yN?Mn$WCY{^g z8TYLDWh4>#^+f?3=@X@ipf9XtzJdfHjoU$5XxWr}QB7uT$B{z1X0xT~FpB#iqV zb|52Sz`YMD&| zY2~!4hZ?0r*S5i(CU_bD^+ESXhRY=y*g0FpDHe`uZF!F+l}#J?!gtH;2y%tA_hN&T z5F#`*g(o}(Z}7!Y)AI@fNvgH)j=I%uAgkZyyS_f1sx?$v%Y%q12dw~WvX<(MB5001 zl~&GvvycktG+^Kfeb>8P7WgPo=gs2pXdo9&^|A;NK9|QaLk^gF4DV8!xMBq)9e161 zCqsPCKCIq7K^sn2YZx-qg+2Jk!R;@i@5X40yi_cd#5{o3sT+rf#-6m-%pqsW=_>cj zBVLHNSJU6n#-Jg!-Og^MQmJJt)e~xY!)*TS5V9W5j${wGNY3yXyM%W_HTtNsTeoU6+TtkNbBYc3w zyRNz-P?@VN9Z{tK*5btKBH!gL(I10hJYpPnigQQb7=tCH>`a+IGlz}HLBUp(m+BX!ma+0PR+2%fqn_8#<0+&sWkG_6?_fz!MoX) zTeF6vzuJ3jXlCB`qky@b58N~>0(uBkQ- zCA}~5`uZ+lzu+%i=0@2J;{1Wjy{iFXPbs-^$BS{5SG4&Ht8{nfv>^4DLUXmod9)|3`TlsxxeG7;;V;9F@po6C-t%7~A-mB11DmwcZjF9=f&-}c z^?pbUntvRpfrTERVLq1E5fRg!2^517^fg@GlY-H8%#&-cC$c4y3N5_Y?P$;)6P|C*Qi_OIn-tbfVN)ci3oBais|ybSa|nU|r9S`PhdUM72P*QRie zt9JaCyv%_A!N2EaGVgC`{w*)F`yb_Hsuy1*f62=r+x{^x6aS}q8KhhRpT`MOvx$qi zq&8HakGXA_PrOT+O=XJY=@Q*N zJIkaXkyx+EGCDd>Rgs;gLzQ@nKCA4w+KPA0ELDrA!I3tS$d_uS8PL_X>)t4=kF#1M z+UISnZ<^s8e}>xPT$}}K=Q%NXX_>tYn;)(L(eq-M1+%5J@X~OaWd0F4lZBWLVv7zK zq?YF%o6$Xf5ghZnSSrhS4lpq5uK>Gp@H=f1(NUET!R9P&;Fctm_p+#yJCzc1m?g0J zxQ%D^_1puWe3gjYCc;L}(GPM9eAw{>Kj!T~CUPX5DR|KWvXf?;!3gDuX)Ghz+kbif zgaOf{3pdslX`o`#`hzjQ#a+brSbxN1Ev5HnlWmIoTrlbv7GBRkMyHOJm4G+nrMc4^ z@9B2+$~rZldvXjB==^W3a1Kn?a;G*tumS;(0pdR*>9;l1uGO!yWu)o$BgZYf$V)=> z#tV(goU*WM?0fJu87eu|KF;^%Iv))F7?!X#u!8=)HT7ORYtkfFtl&tIuG896>SRg8L)$q}8e zJovWAp`Jg?h1@u_xcuGQBSZf>WmO$13NU$O$Rh=)Spm}`|LXJFVF*0mdMJGM5Xyy4 z1ghT~krz0+0(`&|z2!VPtzqZ;*_E=2YY91)1W%!Y4pyLRhbB-hBO`4CYnRTB)}3JV z8$vn+iY8{yy81WqnvSqV+9VHG@e*w{X|Ln)S~PElHV2PmtaQ+tEI4(AEEwi8yUwsa zTNIby?85(e(Trf!F7bWN&-jAXIH!N)p47$~T*K_{;}h{Q(Y682n40ZbC#(F4OevaO zdKc8oKPF!^gZ@xm;oD-W9?$o&aR;u44a3*u6RG$56u%Nn)5qsRNXR})#QWWEi=xvP zDGo$~D_m)Eg^wBp0PAVw*_>jO?bYfl+*D;3$oCBtgu7br_!w}>^&XUT+LNDeS1&JO zDB`hrt{snH#K%N*Xd$)r@4Mv1sWLw<_t-|&QrS#p2UT(k>nbELM6BL7pqle~7ucpC z8KN#xHu;(dd2TrENRDF0l9Lmc4cLcQt@dl7kEt_Z$u22i@duaftIc{$#EtoQBPeCK z+n+A-82W(^+Gn{B?;+HNXrCp0Bzr;rM0Hwug!7GU?)Ipu1I<)@F9ItDAoHOCmu1J} zLcUTXK^f9hE0J6iW+9&3uuisk1as#~3yV!2B+DhNBB_pKw0uZum?S+lG=6wG*ncq% z7{D;@A7-ooag$XBR{Sco)lRoXlJ7n)IRgd{HX8nv6DM~UZFL8Ezl-yuk zv%-yyoY179UZ>IPPu{=vnvEg%tQ5I-3{H%C;_jZBG@har0QwJ2&+ zcCY@tH+|H>ah5O4XgQ~rm+fkU(9U2*N>(_;TnuwW5PRG3SkYHlG**b;96gj*c~;qK z$YN#hTlZ+wv^40M=fJP(jepVZ+m(g&*>h(mdPOHj76WGZ%rU)gIq8;)1D{$rSfF(hoo8hm(63Z{Z5|1GDufS5ozX3~|SHA&E zhra?#8h-?qKK=?U@%|gI)VKWd3w9j(E3h7hq}UulpBA zw&9(tZvSuoZAN^+&X?ffC!nFAUhK|BoNJLyP{mWMOVvSb*K1xBssGwG8AQQb>-PLXpnEew^rzvql&yB!M8(heK}8FFfsK6-IQT)0 z(I7BJZB}*1tT6h#MZ*ZnXK98JghD%o+R3VNF|Ao7|+?VX#9Bg)`foYZ5d#`w3 zZUwOdbvzfkTz;OzEC>L-r`Dj4djtFRJn<*>4&E~MSR6U&sgfg1_(X9C zSp@F4_uYM*>i{gK%{+kdcI2!^J;@?E?Mpkngf!^sZq7v{r=tYNhxm*cYM1y4?(wgp zLxPq7ngkoxwD>vSDE^V)ELpT5UM4Usemf0~@OMNQ7;pz|`N}%v`Qk3(QfUqJq_{(_ z>vbcRc87exJ0ak)uhe^wxBva)pPcb-CURl!t1N!Pyu8py4n8vfnP8x%Nb}tq1=M`U zAaNp;On2x}`#qKHNQNP-TNIxrb4UiNJ_(YLcBA2DxtIXem0QVYE+xxW%?f+8-ASOd9wF@o&-%{$3K6)AX-?rh|~kypVy z$1yP%J5vr)D>7<}c(TAVHm!q@tdisQ5fs6O+w7s2w8nC$-Fsi-wPcGz#pvmLmdBwy zTLmaUJiL*n;^Pugn%@NmqWD3a0oKE!h8d)2IwtBc1-Ubx`ky{f`sWAX1kMz`gu_vE z>^5EKG{6`fBk02Lma}~>RNlbz*fvo|oIRz%H6DQ2g=tDGb*F}SrzSE*N^gas^+nRh zeq3NFyKsG~&B^xxgWGs%pi34bumSK%3;*qJ1=gC#rGqMNe?qvyv+_+oiqIgCBkMyd zAKEbtNiwTK0jOKz?f;FdEa_UQ1Bp0Bt3$#YMh^A<9X{ zce%_|Y{*E&{~qSKBenQC45BOgGSEObs4ackbjIsS>d0g5sp>x*tAx^pllz%fY-;p^E0}k4b-oQFKSN3 zLl(0AuuCx*mmh$J;$e;TFy~Z>r)@Q7gjGQwMo{xbE2^@jOLQBMF0Y_3UwkGS7vhca z!(9%=2}h@)Tsrbw9stqHtEA~CDLFR@BI>p11F?0bY!VJvU<$mb*!YJL6z0$5Ql2$) z&bZQ4p^&|Up^NBY5f~#p$`W~L_996FD!3w^LqXfOx}c?o&j#O=G@t&!{&FCzr`#S}ZZx)h)$_t7g7FH3nXedfpR`-*@gGL@7NIIQK} zCfr7pY%FKBfRdu6iGo);3Q6PU)DZZtiMpsd02Y*GCyYOe9fmM5XOAJ|K0UiQ7GyX{ z^(zY5!H=x8WPH}=)93*maPF%KO$1s@d`RhW(R5@hs3lQeM)#Sdx|m>!P13Q{%w32`EYiBzR;Y$a{Q+mAB2!<A0^fAXh_M$vzPk5R1wJ9`YhTB1*d7w%FLf$GxY#wt$4WbVYJ zmrx^U?;MIu8X7Gjgc>Qd2utL$wDx1U6puZHA{kYT#kC}ucJ~t<(a_tG(D6%|j0m;y zwuat`Lf2$~i~icA*!fLkhHG~+6i0)YWg7(^wi<2;AYQvH<5ttPJV`}|wBn&bEM7g?%?lYQQ zKwDNl`x=G;cyNaV>jAG)r^5E80?L69l!i7uiV%VKSx1n`r_X4*mEHYLpq`%mdT8(MIzzl_G=){DYMa2p|Gu58}p`@+Ght$9GZT($8TP$c4>&R zF_=Tak@V4s?v5VR<@F~k8){2?U~LY;QHu|FHtyX=V@Y(i|I_{fR5X|{SY8(u*z0aDT4}`dD%nT_pGDoqcA(S3$HR*V9)aP-1dP<{mM_` zbdCAG^X9G%{Q{R@A4_pyOAE}%?Z1>$USj(+NM4|3vH4CoiG>jhE+#Bh;alh?BK!>W zz<*!%5&jw!hk{afIw@hFMcZ1w|Eo3LBEnq+JPL0qdp`Eamq}V)dwd64y%PJQhNFnU zWAu{-Ff#B1*nJ<0*%WA&D9_%Ebv(`5A29R;6;zrEP;<6Madb>2X@%plB9fxkozaQ# zHi*)yHc4SyP=oZ1eu%3C&dJkBd`!j0k@KI#td(YK#L=YYq{9r<#d(7VWN$6;S0?s@ z3ZwnU_oIlnt;kz2cZqdr@po}?wcc^R<8oy~MCd7%WI*hIY}8|amrexrfYl(4E5}gCu?aek}G?(np#$esSwHfNmjaL z8@ObibF!6++-`Hx@eDJSVx1DwF>aiIM56cj?z>C$$zgMZLlhsjjehqL*KBaR>=xoE+!USqSaqfB>~cD2)h# zL#>H=xI36cZ~cV1X+Vs=?+DuILzcNQE_LDB$qaf9^on_2{xq&JtbQ7T;dVG13e3O^ z%Vf_MeR^ms44)6$hm%?X2OTUaD-fBhg#9iGwBcnxVsmir5z%+(CJdL8C0mcxDAf*( z%g<1Q+vmzeNF=_nx$lgmoN;FOLkNpRdR@AX(T8~~t71GEf?K&zu5^(Wd#i<^1;vqs zVZ>#W!u0oms6m^=LgTx@>EPddrD2q&2HC#hm0y}N-(&AKUo1O^?}YCD(3HXc`=*TQ zFHIRAiMOUq{x3}#z1h3m+d-k`Y6CSvk_4^kdyS-`d!3(RKlPB*1ClyFCcc6Xa~>}D z+d19F&5EBIpT!NSMdIC~8BkA6=Jp3!R{^jUf#@uv4Kd+w|#jIL-IDxzTn30B`c<|Mh>8W)2x3TgRf(o>w8A~Gyi zY8OdJjfyVLw2V4MfuW7kZs6tq2PT-I6xGK@DI^qWvNykg}+Lx-K(*=dH&FuhN=Nk`91Vu zZQL^QTB4XI1R>Kj_3O7@ihV%2zHgQj)Ly?-M&;THsB>t@=S`1~3>stbziZ0mo{kgm z-Bhfg$~-?#4|&n~TRpC2*<;}v-@w|7gZ6)h5^R@EXz<-vhcF)gS`9t009-T;ycl^` zX<^3}Q)8%|%1K%p)cW#W$tdf&3RMZNPuwcri?AmkeJT6=)~c?JH8{zEQn&B=s}0k&aHKktm*Nf zN~g>eP*Yy4=&`XMlgtQgPKZOPfsB{ls8=wv5ET8^L-+WY(HI<8%v#6IV?DY2mi%%o z_2R;m#4%0+lsVWOr3mY@{^`!DP}=?hFcea#7g~SoD-r#PuS5?`HCoy~s3pk&AdONU9Udt-`NwI|dWp>4Aswe` z-u;|ozuqX_K;C&x;54Q!i?x6hHy_6;wT^&)Anml!0&G6PsW_I+1$~n0r_X{2%HJ7x zpdubJoMJ96zM4v6&4j1R zFR68wK>7kTP8Ut4!;|*7JXBYQH#_h8N?lLH8QJg2}-~BM@G7P zWB8CC<1dKsu9j8390K_G(qo zN|thROl@>OXPng(b}e8@_SRd!VM1y@!=vP2f|QjdmtDn^QKRQE`YS%4dea z%^&wJQ>6d-{tKbn|LtG?*Y+>Z|9Jl*+#Xu$5jvT+V0;d)x8PD&Nm8kjeYe-nUq137kZ*cdFiQu)gi~t?mP%oiQ4^Iu`iv(^+(L zaipchj&Q;8g3Bg)>0^9U8OFA#O>ynTXLLVwX}V$T!;`|@4={}87v)$WD$(MumU!16 zwSJP0JyoD#DcrwKy|4(<&|tp#rT~(C)rP^}%jj(WN!O#;&}val$_E<6RmiWvID>6U z$j^n39#Jg?($`>TY*R>L+eR2C(|vs*@$0||y=a5{N!)((LE~vV?MSz|Q2JR~($Hx% zr<9@PFx|^py=7|PNqg8`uMK(CXIIRDHHY(w^6d7~DA&){niCBD+I2hV&0i-d`O>-; zkA5sgS!RRmQX0%7wCHv7FDO5%vURN`LXtxA1s%$*PMzsXhZXC#5z|n7nacM|g?cg&w-fy>K+0U~PppUcD%k}!>2`(Cz+-9S9{Cd`3ldT}kyLa~XXgsm@w{fJD? zVX)9=^uQyqE=@VMg^t@sEx6($?JAFscm9# zA9|n>168O`f_M3i@nO>4=JeZ&{0WGZd-rxd{_QXeM8IZs0Bkd|yJv-7?up5>12ir9 zlHsOcr+&&jB?R0ABg4I)(d9LSDF=-Z2a;L6Nz+^t%-BO7YezZ;$}NQ!v;iYeI|sE`nI9*kWJ0w4~2|f zXy3y*Tqu;&`Jf6(V~d}SclL*InLxN!?->RTE2ssCG9^a?uCLWT(Z2ALgin7G@OvXk z4?KtMtXguu{w$^Vlv_;8Z{C|w>$1rb-as%xzIxwhZTdue*GIl8ZxOqpX;jv@xC8Hb zG}=eJr}ZO~gMq44xsBj`?X_f=DS-V)V%kylg+P)lxbuhCFY<7c3*>{u8{I-p;*#3i zn%|P4tux20eXSFLTm?XeQP@TRP^s2v8hCT^@UP}ynEo&S^8b(g%hms8{)JmHImnMu zz9XRZz#t2zsmlfTn7B7h?6K3ZHm(%$=(-Lt)8UT+9m?flcwzAMD}3P^2)5&teJh} zz9y`BJ@>mqc@S9gsym<9vY?Y`TAAxRdt$RDHTG$x8AIdS5>M8KXuxE^FZY^vlN;`G zyZojkk21k$lXi!o#^agdRgZ5@`9D6K-0v+Ue{@a>NooW= zV7wrSvXCbM<#f(gPQvelOuqVS`X^2bVSH2;d{0z{sS6YiO@oR@mOw);r@T%^7zM zzL1Wv_aOTImf`Ii#Ty@R=T9eBFBI@`98VZ`o~>vXvk4hG(@Z|35^17VdY@I9NUeJs zR%Xz&I20bf*QvUR+8;kX*o+PYTq=ixRwL=lsC$Gc9Ng3gYFg@4ih1}p%ACC2<<=yz zRsRa%!oX1Qg|-=guE2zy#d|#8{&9G$fcdcr0JZ|f9|e~Z`+)iMq$#%A=F$|dbxC&7 zwvl)6O_#I3Ey-uFvW0hRjMB=%5lT%U=E)HPf9w=7vqpzZ-cX(Q;_q)TZ(OztH zp?~@iP*8xIyKERpbn@0W^FQBp?`f&bm<^9raIZgDo7kS{1 zcrNV&;-UIwn=a!=%_md*p~!7IV|G1<>d9@eLLSPorj5S zWl`>(lA5QzzaxBIEK}%)ndK1OT#V3fkj?j>?`f*wdSx`U6dzA2<7oK$$YPs0b?x39 z!9lN3O>Crm-Yn|Y%euddJlNQj_whisHhDf3*DbVsbye&nMn5|hhA(q_dRBfN+e^1d z1P^)S$rJ&zPhX_erteG5+8B?P;rsT{JBQ0$Gm(1NY?8M*&#_Q#H_-FL%(1}O+sd(Z z^(o#4ukC;>K{AWCl3RQB6LArb9BE&(-gh>-{5`l63mM*_+Mf5itdd_YbL-zzKb$u2 z@8Zf3F}NIV&Z0)71lOy=)wTTe%Z!j`p&*6g9vJYs?BGE%>zOW=8bGsH_ zq9N=ctC6#L7=yUz*dB;YqQ{ex6jY8}Mbc>wucb;UHw2{AWCJ{*RK`n-&NB#8@2+t^ zF(0q2RJ(ROLPf2zAIFX28p3{FqvVr#B<5FG(VAnavn)BCv-P4JDjPv%z|l#1wqP?E zIN!^`5)u*cANaSuU2p3sVuY(lZwNUVu=eH|k7yqe zI7D0eS=qYt=(MgFel0Ay26lP@JO8z6eyN9;uW?Rcd)en9Hre8r@n+pnyaTdRjQz2aKroToOS$u zfzDRYZhf{%K-s2kcv`^sZTa$stMhH${x20^Y;*km`*y6|5{r2B9OKNt1DEH#U#DNM zUVC>Ry5py@c#0)@U8?ND+oL5zzXGE2>1-u~<-B`a&PT>N38GJ&`=|kNk=&bBOOZ*@ z0!a-SSYI)NNvwB$vO=b?WWGVB&99G=M54j7a6TIG2`;88q94)

  • #U`A&E|FMZFQ zie|T*!r}?5r`=;3{!S6&a(0ic@--Alb{{A7=(7G1OFBNQ5+>;9bQ^h{&c7(ID) z?HK1j7|d?A4lT4s@4Ltiw4|1*_2<7R-EMOQ2m;@dd~CBJLA@(R>k7nhUZr3#AKGs6dEeEH z<1%QYb86cQqrc#p^%W3+U4P+`F_UrnEPX8}W-x0q+Y_u5Hv%24YiR&wGrca)b*GLx zqd4g=8aFLH02}L7aq`LB*~LDP9X%L^R0fxLTY;>wSNud)$1&&FWVsFO%Q?3Yyh7>> zMCvrfyKge%3QnowkQ$*q5}VF=!$%xx(w33$ntiuC_tM=Y(JUslz(`ay>h(}oAS&wN zNF4|EsDwXXxM}{RF@tmS%xLI;hVS-$%OFUgAl-aRBx!)16icg9M%A%o;TO(g<`gDR zc@GNrlvC^!BTMr5H2v~v^Xdta2I5@8zcI{wv3ZU5vq7=I+ zv2FpgISsuxwx3lNp|b&NjiSk?;|(22p>&DF9q$byr9VT< z6gBjDI)yS>@+?RC9*ozYnSGOgWj*1Z@(Fet6iM-=w!D-NryMWLI*J|qoLY3%%WYK; z;mdN)xEBTQ;}(%Z={vl-&i9b1$rL_KHW!>Jc9D*?;IE9;Qb)~7kdlv!gM|L+7(?GyrcH%k|1XfOE z`oKL|IuguZR!%`$c18BCw_j7k9p*qpB>!CvK3pmXnk6MXLF4jA4*$a`#_i0~Q0(+>39^DgNp7h&WC72IwOFU`UsrE~VKwnhP`8}8#o|dZb z&R!5^2zJAk>M9~gy+9J_gh-ZCnWUg`)@p=lWUW+fU^}f@PfIH1c!Q;0ko=n43jSr;VnTfIE3q|~`b?9sTb z&JGdOg5s~;S}*W;2u%Yl6kHHB_6ElqqgCWkut>^n=A^BOJLu9bCJ6nb)snre%QCc3 z1ru?fOFPIe;6Ax2e8pu7MZqu+tEaeWW=;R*5$qV7*+ox@3;OkS?mJU>OOfQHD`d(m zD1ha1haUM55x4D3x!{pn$K8sw!f2d94{_mEyP7nH9I2bT5`RXjt&p40y;Qv}`s|oJ zqqm<7)H&TEQ3$cQLL`*)#XP^wWMtdMr?|!D2R+7=%jo>Tp!yHSJ?S#3e8Xl4(C|c# z?5LCo#If0gbW2@$V%8w=Srieb6ky_)3W`W)bm>+S_QsQN1W} z`c?wQ5z>J<;2ID%%UBf4=&7h7mH=(4)*lKT3qy3;+v2|Wh=ryUbX~qxjCxX7<#YLk za`Ol9Lx`+~S(7B+hP8iiQd8rx(01%xC;Fv$5RSW&5=;i5?g3zNJ$;MQ;fs!0aKD6T zB8qsKL*_eY=?92HbK%2i`iQQ;WuTytJB$=c2hX66%4~>6__yk;z&HVDF%d~kYHFIpS;4cAQbN%}_Y_bGqxX9}+mmfiuBmAzd*#Wt-DFO-ZBDjpvTfUwCO5gW8tnb-ecp5N zo}2Dh7mL6D=NrWdr_(T*=OGxtm`T`HS|PY6fNnm?T!n!nS~t-p{eJG9A5sllP4#!T z<6*43FeEMTq`I+lmfMToedRB-KMw`LX7?T-aA%%(m}mTyHT#q>D4OV1hVshZm{9zuP=wI3Ob!yCC}Dd zwQQb7ZJK7Wr0e~`%m~}XK4A`vOlD@m3MA%uCdsz%Tw|WfPos`PD{fYFF-)cKYoa4@ zIj8PyP9?VH(M+f&qzCJoMv&Y!>Kml0sEJ~mQGqMhRjBuX=*8@rg1q!_Ym;?S1~KQD zfSBNYdcNFwjp)jl^O;p7*N(U`e-qsWENYzb5FF~e!XGOhkrpZ-upqM%#b|V$P=zHl zfNrTtcBTP!a{cnlnPv3nRS4Z@eOdoHkn`^B{V_V~Jsi5u24BP&k~~JKe)dpTKM|}& z!Ao%_iGdcesJCcyxN(h@3O^*yw5qw}Ol`g-0O0T4Tj`|e_moPse%`)i+*PMwo0d1q z0wE0}Wy`QYf`9)3cGutkv+=oQsf~aF{`B3Q`crc%#2ZWv1`_+^0HoRRDPMAmd-)Yu z6H;t2p;n1DFeg2Ol@us4XmczarD~~A;C0yNuc|$M3Pj`iz4SV1)9ws4`}}Bt>_Dm= zp|9UQb^ADSrlfY2R3IvJH0mgrP+on*q){+);V5cK(N|DU=#kF{`7~-o3*ltg5gEgA z?1@2aSUt&s?I1r54N*11u@oYH-_l{o{^cu3eS-X(BXfheZb=D}+0&)?B2;QT=`fiw zxwnXmoDzN`S{(J_>|D7d_d}Z~)awBwXho2gf|l*zq(ToIBE-`5X`>B#rAo(q#77lx zppdY9-_pa&( z4T)-UyyIRWW1MgaiMFAY;FvN>RU`GPdS{MKl+D%Mj{qEoZR#Vsj5h0j#TW!3a&FKK zTx#MY^5&P%xu%CzCG?H&W5ZAXsyk{E*$9$tFY(6pi^XzU&UF1|Lbc&i0#9lLb|X(G z4;?X!PKIooD5;bSN^THV*kYOPb#X*xSGJkntN~N&&2Hg^o*7s33bp}x+*}KGQ`Vd} zc|N5njFs#+f#vziQPg*L^i7xYxCvq>TyZz#ClchgY~cnA!oqp1VxdepC@zu;3(YBBQ;Et^CraM5~&CoB;6k_ma$Kj#kv(Eb~v&4BzaPAcA z(i3jJg)~U?BF7&tgyiOClD0Ak`fvEi?@O~18OFK-vj-*Kb}g+!ASddJl0(z=(Yt&D zcIK4(OF?uNyV7h7y#vz0_}~Sm9ytOnL@Gi3!{j~yK(k+Q^YETbJ<8>X{n z-yo9KU1bU8_;BQgWU`3(Knp(90Z4T|vZHiCl_$lR$Aw1<8MBy257eR-`*(^6xtl=s zVCLRdU)2nFJ2Md#;NZ|-kaXDqKL;`oD|ud7O#)_XtI@#zL^^3oa>ZUxDG04+^M+Q$ zL4`^doqkw@18QKPH;{fZqe2xk!Iouc6zOB9OO_@S?tL0+LWkkt28Jgm_!m9|f@uVV zHapd%(8?OB&Hm@5?F2(xKd2}+WYr%4QrooEpK=`DzVr|rBy*(e7I5Y3aFd67F{9S9 z_|HhYn;>0@XRtR8=z*Z4f(!h(==A$7Jho}}eM3R>EJ<$Q%oV#j1S@c#Hw1S;{P^Oo z3ds{Lrcyu`i>4(4(l zrcEIpk_pKR(Z0(ENxis?bHHn3;hXVs1j_i7T55)H?}y0;|3V7-Y}KzX$9Ep>`+kNU zqaaUT&$>uTzu6QvUMIHMqIeIfcQRYq@RfEQ)sulH7|+n`;oxl$`1MKe7ejnrKdrOd z#u#h2Jz0zB!_0hnu6gGF_)>Da{_AtyD*xL{0+%;~WiWG#Rqk7qe?O=u?9Mei3JYOz62S0s>;`4^M z)}y-_DZkdGOgNh|VZ(h2_|e_{^t?Ho_|%%n_|sLF6Um3zc@AC+-!E{y|4O)BeYEbn z{!#uxr4OLCq@*#=QBplL2V5kUl1=X)9 zlI^WK*(u!s@{!k%TEjssKC_oOdOt8~6mD~>!2|-Xs)|jHZDN>HaIrE;(?IXeQ1^=K z9r-jEbdRq!jR6T4J`^D%11tk}eD|%bz8}wh8?_9%NrSYXfUL~yzgU^~|6pY>E7FFT z4mpc~tjyuM8IYARwNQ)%vNF5yB}0@u*h$N7&&T;=BI_WLDzTq{%FyFxdtTS;ONxLhEN}mNF%qK#>(E2U6 zN5YfGO`WxKsElINM4|U>Ew;i~k{zaBEya&o78${=j<*Nj@a0%WNfB7W`@>b-40PfZ z6~Fn+Y8xtMcfvUHs}4FwA6Dp#^u|^+k{{5EXQ{zukddHTz76-lHtuzh6(WAC0A=0{ z;%PV3QevWruk;zZ9VR7|!q2EDXY5d6pbY$-4_sy0PD@Hq5GvEH+9IaGVWC@yHjDa> zg9u`OntK?0ct>SfM&@wm^C%@f1EJL0Xl>*!k3nN%)i11@N!OLC_OGf86zY>hpVxeP zahd}dw>Og^vrNJqei>a~Uc_Enaf0aA_a$1VFcF4s;ojsQOQ_a?s?5nRRfZe`sLF`= z1_k$m-~hf5`jBtc+O-!lWc7s`o2tN(NQYj%iH4XHYyY5yc|zr9$NcGSg#AWskkW-Y zU}fKXg%-u0%TXvyDNRYHoDhu{8=@IPut6;X^_MCWK%NUzLJE!RV*X2&G0#f#GC){$ zMbP1wO{q~i#$174BQWR%`uuQ-;uuU{^{T++UR9ZFXrL-XIw7pV=uRm8N0lMdkKQsO zG^+6|fqA5X8@fOs18^#Ab;5l5)a4RwA2RXw+6jXy|I0OW5T6}6s@EhOUyLclP|Zba+XPFe0ZIP zX*jTkm1%>oNhJt!$!b;daftmj$kqU?Z;-7%oia58ZkaB;XAgo4e*Re;De)-rZ`IO_ zP#Jatk5jZJ^?uR`wPs7hyK13mLYfZX(6$|a-O|P1`%neR&7O0$6&IgoD#(E1&#!GJ z1sax52vlXHz@yajHZ|2#QH7LxM+S*MlwTA8RhfKFmiAw&%m~{W8x6nS6Allk1O;3 z+h7poV%@D8GBJj!drX-5b~c*_OEiVzz#dNSebAV6qHht|zf_sIZiB{Os?7J{_N;AY zMBwv3@a0E}?nOl$RmR|tDl>(A@Jp3JIu~jx3%^HKgN0UfGd+`tE*hm;jG$xpt8iJ%tlgFp z#qGEa9{-r%3)u)$5!#sdeq;8Q)JA_2)|5cDREntai!nh{y{iNvCzzRJ_U1bJXysPvvAjG;L2G{L*?7b-It0z_q`fT&DDS)S`(REFcP z?$((yHT`|9Nr9-$5H-tE5xZ#|m@ibmr9%(-SKNJmUR^r>1B{_V<;1ajmo6iKBo0~8} zWU?{hnlzkUl~*q2yg&BP8K(b#HI<-5+mNs76fUo_{IU@P0dNs(Xo+;71@TadiO?)F zH4Qd_wdZ1u){k&hGDZj=8=FV;-<`NB`VYw4-`6{pwgi@-revhw$kAImY%E(2myt0X z=a)?lu`E}%giYQH3BJdb7^j}V7BWQA(p`zQkb~qk9X28JnY*~G%gLqIwvp?gvJytZ z*kvx9t06&9h>A`R@Ta9=`;x}pin@nN|+h+S4SO{%8dI6_l?e5Q#|0v8xAJt~g~qcqZvCLX{0+f+*X-Bc2P zZ7O*Hn@S_!of1E?LRFvQu0fZCSN4tJ1U0R$9ejG{B!ju1GyN@1J)labm&z(P8BqV$ zqilM%0%M#5v=7i+h{M$2=`xf(lmFGD=`Eaqog>6#Vtx8*8wvuT-iCLO2GVq+hd}W6 zrp5yN_ok*A)-jn0fk6$TX>GuAExr8uLbE=}OtvWlL$tn3hWGlvNtr%L@CJ*I3rm)X zfFB4c>`y7_XyIQ&3#-;9-tOO}EKiLT)%Sf?!;U4@G#D&%{>*sr$bd43R*b3)C2DCV z%e7`lXpkdV{;#A`(XXV^V`k3Zq>}mFi&+)*h;JMKKor2>^5Sv!nDyg@`}--;hd)In zi@!xB>OV!L;y*2@|aE_gDdH>XmSTQG3a1ct#w)fto(|Hhi(T9iJLFC}Ae6-_1-;+AJ*8%UH@I zrg2+NH1n-G0Pb?sBD&22DpUA!UQXBV~xP6M4Pj{b?UDjmYj~T0bhvPHLu0{D+ul5tq~N47MOoiP$^Ugw9Wx2zSLd1ooVvogXgHwK**tiM!vjQR7EE57 zN_ZW(6~byd?0N~xkjlupS}QJ`O^5KFW8^EZq{;#8t9T8;^+E_oX+2b@i$86I6m$G0V$c*)uPr_)4_R z;=9NRF+l|%f=guf+9o7i*%xg?j#;Sj`{$jL zRGHHVPxs6GMHIrPk9Q}$Yme2HI2Qoy^FrpYbItcBk3bVBV*Q(I^~Z>5u_%s?6$(P3 z$uL;*4oa&x5_W>v`z(aYc$92GL5VS*qN$>tEn{qT!VH;sUOc|323R30v#CKt8WzRjC*jJ*9js@XJ15jODh@_HTBDE=MQc>KqxhJfvV ziYg_z{A*M*(DXX0Q3Q@^FkeSCm%vd?RHwHRbUA4;cNgV_>mpxCP3delH_sCJjlx%E zx+wNy6R@TPOu6N_<#1Cu<`nGK-3g8-eM33yCpAn^OK?JA*rC41kf)C&*_N73OWkxX z(axIIvi25lQ*|{5SI=n6ks3e49xkVLD$x9H2M$15r``#+#ECT+1V+t4p+TIak96GweNw@L!PaS*Qw z^^`y&w)=W3!z-jlU}{vSw-;;-2KM(2WyjtpHUy_D@Vg1TQzaB^imvP$Qm6#}P1yW_ zrnG<(^dfAq%{|~nr~L1z((%7g8H4``l?nMLD)a7t6;)zwJ$E{PPo|b2RCW+P7pXd| z__d-%xK!2D?CI9WwH+e|K7oeFi~4IB#7SwCYkF3>MlPM?>KdV5!R`_p>9J4%FM!oW z8(lToUzLUUG?G~X#vU?#2zI%d1-SHdSv;X!Umo=Gxek&ZhUhfB(0*}wdg|}ota%c> zJLP$;D(QaBHs=91%`2)2@x?f~(^Z;4Aa!qX;30b>QQl#dkf;9$XHh_*Z$S4_WabpJ z?y1g`k+3sK8u3G@i56xuWQ(%O@K}D6taQYDil=FAh>E zQ5l8ssZP4B=DdW8us7qdWL9Zfkm9Ag;24c3Q4Vgmb^_6dy#fctgFeP6b?y>}5sL{f zGLf6GHN|DC@B2%vvl^vpek3JrVqgtxR%;r)j8)fquoEDD@ng z5JVIMAQx{R0wYce`mg)SdDuw@S4;&}DsEB_^}_a&rrDXo7pqeg-Y(8p7Lb8KXTPcc;O&Y3>U{v0Mm5Wn4+E zu)fxX^$Qu$p~qS!F1o-v^b21cy}j@-kvaelyMx%-0J&B>ZvL9hGtihi4 zHc5RhPU)CJ5G!65i2rv~$@O8AlB498tAdD zzE+hwYk->?t(UBy&d;aYHNw)q)p?Lg_1&5xxm_lhRI>5x$X&x5J}m2~@oqSni~4?2 zCNU_UZIk713b0!8vczTkl+k_i>P~&x?CQlN7=fycBa(QWi62mvY1r1+JfQc_z$cIS zL_H$v(eLH)eDG}q?$9DypE_0G+0|V-r%@-u3pG*mmeJ?*HrESX^S0tq&u((}CEBaH zq*fh~4NhR}oS1ld@qE$Y$MJ3M#?<2ZFwKjTQV-VHm=`e5RKbsG%qk+S)vgGt8h&M!xQ}_4S;xIz?$$2<;mVMk z`wTt2)lnJM_I%>U1a_4We|MEMq+h#A28gZhWMw(`YqGb|v`qbi4BCgS9 zYp-1;fVj?|uF@NH0M2h$268eHGJh55%Gd|Y$EQ`O6BUE+HTytOSiW|Zd=*-uk|!Fy zL0SvK@^Z*IwaM@Evl&deOgB2IM0b3J%W4?F>-Me6Q{1XyObYgRN&1b+jmX7RC{HpJ zZS4%d%)IfnArZ%Y-PIs=!LG*_uW^3rtyYAzShw}uOQD68SKyhB(xaky*Qtn>_=O%~ z>>;9te6YMq=)DwIiy?SFS-SyijJ$ap8VF4kMGIH9R2D5A)`f_I1sFQsxDiDt87+Tc zAlLFt`&o&TouFF4q-;2WJxeNtu+S%5@qdSv8mGqivokLdBIF>2yiWgwm3ntAgeHl~ zbW~YNdU!FplbP?F6G5rjwv;Il0gXC?vTdfg<=$#;B%Ug%ObRH1z_OAp9~e%$fY1{(8`W&HlR;{JK9^Tj^xl7(90 z**=ZKbB%&Xx+X|0J|Hb*9}+{>$S9c_9$0XgirS+%8gj0%$`H;6NLzCs9A7~K z55T+jQgj=X<0kZmA$jb3y*zlOoeqqRwUp-hrW5RQ+v!n9=G@RHM(&$oOEUTf9>3Qi z*P(~k&yxH`H^rqaRH8yOJk~>l1Fejo{ur@~K#~oC71WK4SvK!uHWA(iHMv_>^M z&I~VJq71uZ^8qo0^&)^|o_beSuv)6ZWQci)DhcXxhKff zm)MjDk_poQ;v9CETC}feC4ikD*N8k6Fs-Cui|6?#trU!aJ+D$FhTVI*`~krAo0c&Q zP}10CpGy?%V^l;WI_x!zJHQ`O)EiVi{2qq0Mklz+ckQq#lW=E#;>$ebmMDU z35-jufy~ekbvKMmUL<2KeZyOdi%YZ?VXJ2Wcek$WM#D0~j=QAmeD>2uoYtEMS~-ZZ zO9LxlTFI9UE+lA^-Im@6Dy3Jw3@wwdR8IvB)lrON?U8b*ThOFK83QD9`&~3Z%GbCbw z#1?i89@u<+@indC9I8JAH1+CK&|NkLRbXlV(!(W0mm(!4NH>iiZKX*CZ~&+iGmvPxqD3z5>Frr*tX}2&{qKnnoiWR zPqBsXYf)w^dK1b5%-l-nQ2I6lGc#uqzmu)X#e_Fcd)EM`HS}q~X$_L?y8(SKT+8In z-_x3$!A(0i|Hk4YI?%M;kfQU@qAAS*ROLw0IxA2xBr-FOc$`hVIAhe3v05}9AqeO^ z5+5z<7c$G_e@ts&K(~KQYw(o67zn6rXr!5cl=FL?)=+}QTfoO0<0Yf|?Xqc5HWj~4 zYb5?oYaoEr8n(nT($Jj)*+~k!ORjAw9L8Z|l;hmD*ED3-=z1*9y$56ug+Jb_p=)ra zfRa(0;L4rd!dYX9&FiyRz?R|qTu1=W$W>iI;RseNh*eJPOzHA8_`;GYBS{ejM9Haa z;Ypr*jcCX^sYWJ%;Iu{nzBnB?t#Rj&(BpWW)=0;mb0Sg1-)j7x*2J*vC5jtj zqNx}>uH>td3*(=+7y+j>$*fW2NVTvqN0|R;tDbHg^ zJSquns#Hg8;>Z*nk(IRgYvJKH?Dcrw5ZtpyY0MwyuVG1)E&Z`fucJB0a0d_k34+9> zP1MjabIsFTt4gx0`%>U1i+S|ANN}S&(92Ezv?_k5mC*0wDr>3`Pt->^VP&etn9h9y zOs-M@XbCJC-O_&JQq~JMlS`*X~6#LI<%@kMk>$HaM*R&>_@n6##bF>sI znrT@OR1s!e8IC|mj10evE`jEnTMi{o5|)UA$Hy|)RjDpmM8?FZCNaUlL*re3G&B#2 zAAbBpIKXKQ-0QTabTtL{*R%$A@#|d{GuHc>4_`HkW%j!CFy;Gv^cbpNr#0n)1v}wl zE>wY#CpK*J{>s`oh2FNEg%e>%E^p-vCoU}HYm)U9R14Dl>`4bECR@{{>u!PNmp!Po2;-kp;2B#Ng?`Tw5Qgn#FhZW_^I z2Tp4s5lOG&yGf$j5ThyNVV6>tJfVfoZY;yRGcA}?t$S6<7C=Sk)+y2P8F1BOl%nD- zLEc&nDS=}*#@`o5h=Y2JN>kpl;P9`st?+gVcxS?57SPM;C3WybBUly^IxMan+eCij z+ouuf8l+xSNwzc#e!N>Nm2)SjrsgqfBsz2xGQ@iZ#rJk^3Rmk{nqtoKQb ziX34N^%i8VletRk2&xjNf>h}t!q@pWd{r$;2GMvt-!Q)}*F<701>C5t8Me?4#1LhZ z2iy#n9w}zP>9;uG%eV`Y|3~pAG;uT;eN;!?pwGC$ipS#|wHd>ZqWdqH!@Im)lOi^V zzik=QKeo(Ape;isI*A+^y;JC^#Z(9p&YJ=$b*p+kz&P!wws?tP9Un$iJX3`sD#|PD zsOb;a2}v=YuE#S)c@1nU8UN?BX7(S`8pjXiJpx+wEMiSuyfDbGY0Ut~Kc+Q(rajFFuhSZTWJf~DH`fwrcRLTIe0nJNL5S6wtlufb zWnZT?@`J72s&Y23(;6Pbo!7RKIF1lJ3?oZPMQseqZLh8P{!ls-qMhgXxhjbSO5iek z*kPq~7PU%3n*~A?YG|ZM@dxdi55-#sd8q`hK9_t=2$icnOhIjkbD7UCTL$qTwv4b$%3oUsu=TQ8 z=JgZ1%Ry=MX5o?E?d9?@OWW(EX5P*H#e?y6R#Od})%fw_kT3d&;fB`yfLJ*4Zj6%b zS%T=YEH}`LELd+P_3%2KICc4W@%G?h{Qfahrc9HS0|?8=JSawyza(G2!ZHT0uuK9F zmYMql%XIx4mWd%ApUwQ(e)==55Uvp?YaLySQ>YIfy1T6XDjXbIVFfP2(HC5gLM!t< zn2w7`x?RRJsFr}q$>(xfv7e7xc-&gMkLEvrl$ltZ-<=X&y@CO5z~=Q``;Xn4{G1+} z#LbY3Oz`Uq?*JVwuxbUj9WK)`JHdwCAF_wW;N8UAJ&$izihMRurlK-pDnPUE~*1R2p>la zgqKb!zVS%|=zNznsKQ5?FEc(Vzjmnqq5WV}pH>Dv#g|sb{hb)(=hGnx({Ae_DbqIa zU9iQ51!`nPLsC8s12*a6ZuUy=#>G$}786G*oTA3E^cYR95BLZwz7$y9MqQe^UA*HI?-tASAU97`Wg$N8dFi28eVzMii(VIp@PSHfM+8Y+ za<4FOTJ0S|m6IF<+1E8k0Y?PNpZXw`R^%&6;k8x+mg>zt!k`UIi}T3M>~D5FZ1H== zEv`y>zexbxM;j=7v{{vu+7*=4L@80+?~czX!kk$sYZM?1dVE%gxg|<>ez%oY)TY8j z6jua7`FF!;jQPdXx1{MV*~k%AV#bD1DW>VWh^L)bg6c$}lvEXa=~SK+l`wF_ys_fq zEwGB7-y11>xC;TqjKx3{=WlSKp-s+BEd<&sX+R4@ObHlPav*F_6?|(6QR>wdb~+2q zYEls}5uraUmG)nGZ$nlHvMkY75*&mL>RS*aw;G#duN0b;7TPw+$yf1*oxDXSX<^r9(|9+Ut8fkOs=Ym=pCy zdrW#wuXhef&VBi_08F1tcCoXq01Y5gVk{w*w02q14N9#b#j`C?LwBNm|8QkGUtJlc zk&;(eMgdfx7LyJoel}GX+jRpsj1Ek*=elcuGY?iQU^?vl9SbIn)t)ZLQA0>aW=Ybzaj-34hW`V9uk~0RcgZ<{Zvrf6_`?ztT#D zHuzk>(@Mo*VRlpsXbttchBnN^%E0G1-bm1e^K?xo*CYBt9P(j<(#R#%6^Z=g7mhS0T5>*e|-K`Vsp zQk{?zd^2d>D=8+phWIUoq#^bmi)LrL8;k*(jD7Lx?1Le}bW(tdkdTVQUYg8tf8mgn8pHC;GUTp0CJ)r6Ay-y&ZyY zfta9(AzW=oe^(Q(tIM7j8To~{eSe(xd7T#YssL8XPK~L3WZnjn@%-fn(T~~7_8$a1 zeO@c$#(=#h+brooS&${V@2Vku$-x1oJb+oLgvPr7hgEe%;>(Gn7svX6^)0rcavZMy znOb+|RRU4Z68f+-Lufe#9H~yCAx07y0e&3A`AsPiw!!k=c~_7kvF%s`4Ls!hgINDJ zxT9zX5d+w>qMsd`@g)g5Eqlp77Lr5`)e%y5D}JneL>AJ1h9rnIRY3}RVBp3AIjhK5 z)Z<%u*FNUc!(KHDCZg!}X<7e@gV1<7ZFv7jgGYP&>Gg`*E@YWr(o14voYhC3v7m%6 z)@qi;Nkkw3Oh=W4fwm*sr>M3(NcNE(6q&f?r9zqPcpKTeM5qe%EjA7~(=Jp%!?bV1lp*55SEoElj4EfuR0mwkT1Yh{%wqWs$^c(7nf& z9&y5A^0|itciU-sdFUyYyW;xISc!Gg3*zOXBIhloU73GT7-??0TC4~SlZot-lBlF! zu`S`%n|k>qt?W|IYnr=Tygiy3WxQ4LN?K4qr5Nd_71iWpRO%WtChV;BGMEDThXYvO zmcY672S1)k>62afJxG@}k@Uv;04TvT@ynb}>|xM(UZCfUjGrTM^jmGFrwfB`gN48ivhL;Z9A;x{7@1Bc`=K zgWJw92&KIhR~NxUtxd?Rz#tI5^9cpxv8A&sq%KhH{v_5{UzB0tT23XP&m9b56qnY< zvi7-B67o~{Dnun$1)ew60!8`2r=#*2_)*sj6Kimt9|bb}=ooQTGMOu565Vl%pxTLz zTR&xNYjIK3j5M7b4A__-LLgkmocb@+?U!y=cXrmRGXYKjiR~NVBeIGbeUbK-<7DP2 zkNGnm6-z&M$1%QN!PFW2>W%-%i2)EJj zsB_M$MZ?Z`qV!YSLc?CqHHptH&cV+Cae!+jiFOW;q`arE{1p6^ppQk3T4zk1s#x$d-PS=Rld4Zx38JevK z^n-)Bz7NV)x>XE&h64?^SP~V@ZNo0>5{LF^EN%bG zZd@+3!r^r9NhUGB$LwvOEJFwSPUc)aXfj^L^#e=zH0VK5!H4OMOK*JO5j(dH3=Zoz zeGWDae&6o}Y{{91IKN~WvsYOLdxT8pcUtM#Uo=BLn*F}$QQ&Lco^}U72KR}$EoQvb z*Uk`&HSFK3In&rU>{fs-#jAB;8Xtqx`-MMj`I0P=3(73$x5 zA}`pksWx4RXBq|6an}y1vA6RUgy=>V(Vh91aWf{aKwK1 z*zB@?T<>KS0Q~WNW$3kj?k-|cq$P;?H%!wnvq7p_7#bhLTuEwOyD2)BAOO$7@&PQh zQ?Bii@Ji2b8K`)5uqG!Ibk@o$S~;izLBVHTd!G}?3w_1M5gd!23J0Y^1j|@8z~3G| zS?L=r>kC23Sss*CfVgkmhI{xjKOTNWy}K}+SSm$epmi?65;_c}-OF3kW}9_`_f4u0 z-97}Dt8tOlFo|6JxaG2~oqIJKYbjS#uKx)2g?6n}OzYz!&&P)YUaz~W%f;mg<~IK& z^IqhugS)IPV@BW*F!Et~@nMWK%7rLO?4u{eH?xooJI{}>!Ezamu6x`$@{M&L?)~8k zgb$XWN_A(e%|6bG6FlPNYLR+r)Xw=kqQLWD)i-cAIW!u;M{4#pIYC-lA+E8ohVw8d zX?!S``_?S{HB{YKWts||4fgq}7%*HptOB;hBj0`_!y9N&Y}5{TuRS%|l6O(;s_&QJFyEGQ5brq?4Ig4(eva%ceC`n~bxn>1aS4F^WWMb-BHuNXmBvgoisQ5e z2ZLt}>Cgey(nt=@^`vb<0B7aM+*EoisBNW5?~6VAaP zH)&tew4MY1gcC@s8FvOc8Gbt&_nB~meGZXLG2AGsc7k>;x0i#5*amB!R_^=r(L^Ib z=c}JP4KL1@!(B!LjJiF!eMxFZhYgpRo;=*1mxqHlz@rlWB@p{WXB1&p$|#W3Mk(q+ zMBTJuXAR4w31U($iF*P|Ldd_gjK^PErgl(bNMh@$dx0(O3jS_7vxmo>sAubyma+In z%Ow7#Weoq)GD973#=Zj)HTWza6)5J3u|{I=CEayShJT70d%0X{6)Z9}d^Woxe)nz^ zBOulkGy3W2$^S&1Us>@49X>w}?e!#nGKr-=?{eJ9j?Q9#w&vB^Ol z`en&gWr)gNQ%wkg!?sJA9_fB_c+Rq7)f}A~eM73(ICS-ZAQZS7L1Rea517<}BSY}A zv1+BjAGap43!jqvO8Mprb6|JW)PyWQS-txz+-stDl_0o$6&WwAdu6T)QH$t|PC<=# zNq6(JG@#yL*9`eT(@IX=ue8h*MWMAd(jQtztl_KJi=d=~uHyX0_P+LpJ>Z3rr8hq3 z7cH~AzFyJ)a`V&m*M=N8kSST$P4Y*my%VmGJ|%l>I1lnjMxJ#4-=>wWf2EatZv6hH zmE!+1trQAOD@DI3mHjKNWbJh-_uP^>?mOVc@>qd7*}#?1`bDgj;zMdJ#G!5qRqxb% zM2dN)BRIePiEk>ieUGZ%mfFTqotx`Be6V?X5^H3dn%b7rvj!2%d*P^Z)cbmoofJj5 zr(CG=X3SgW+G za``02S1W7es%bk*TvSKW?#%q;Z;Tht3^rewb*uF8 z0lyf!%2bL2>~^qa(?RUTA+O@4^qq$FgvbI4m)e1BuW=D#d8A?qnvSiUpNga`oyS*D zTA$o!vF{4PD^TV~Gr!}5wBj~c zC2UKR&2z1iZ8odkz!7?z8Plv^Z@4zC@7z^tX5KJ%euzhHHC@V0Y2@77YnNz|xY%Ap zZ>cRM+MSV6FH^RQLDU^Zrp8oFem82=6SUsTTBgJ=fVrV59HBy@z#FV89{yLB`Ti=) zsQ!{={4*m&{>n0?R4<^~2oi_ETR*gbS&g!P%Q9E5;q+HorZJB}qB?yd&VowSk!v{k zmn@@vS<>|QU$P9_q4@@K)D^nd=VkKta+=E~ldgFvREc^yDIgCaw89{1juRO?phQi% z%20EOb{G-*npS$5*i6}wFzz>=Z>rgj^Z^+6GZs9qtD^pv{7s>m!|+OPb~wl0^NgKTOnTt4#w(s?q>}6njlA_N1KW>)m`9YY z5s;%O!w@HI^3#0$&ZE$wrM|cH!pmtZ>9NrlOt#ou&BSj z+v`_~8iPK`3PwBI{nipi$8bDJB9L%KU~C@xsG-<1{`pb+`l;3TWuJ53P&V#ypVRX3 z8-NFR02WP)$b~TuPn$I|!Xk%RZIzTyG76ZBKi69Q5c=URkP5v5iB^hC;*qnrs|pdAOOpN5=ca^R&hF$O&IIK@o6E|xJH0xmV-84joG{V zCmy=-4x(VW>`-=qL8{hdoPL}P6XKL=6uCVbC7W1NM;7Md=4bRol|Jrl`7bYDkSQmE z{8(tK0Vx7{gvG{(QD@bvocucx#*hk={>*1X5Ni;hZ0Zpcuj@fLF?Kjt%HHZGPOw~Y z^Yq7c#iL{bPxmQwcj><=ID&gyB<{f2C*>m)N6Qp$aXhMQ9Zi~VSfSxCl~?ISGli@- zxFpzJfX`J1B@82!jkN?}*n-3eO+@&AV-h5V6HR(La8|#T6wKZSy4~4XgvJxGFAsfOG%?tZb#@z{;YG)T=g3}XR#Xe-N ze$uprzw4;vzg<$$LDNckqnHB&{P};y!bS_q6dfJ48|cf7{PJZeUVWL$S6{|r#cy9`B+8jfrWt7hhtcoV zm&pP8GNiwJnY!e*5^@}*$+eolz6@;KJ2-TLpu_7n!l3W$k)LJx;2WZc5=E1cM|d2+=D0EV{%Zj4aNm62|@en%e-SDgBdn$l(bV!buI~yuj~rGJ;M{}hf#39 zGht8{#vGi^&H?P~*}wWS3FE{IqUwjAMAoQ#49-HPMQ`MQNI!L>4I8+K>;zlR#yQ3q zLsWUc!2+5ZJH&2$0=ZcA6&U*a#znr$rQIndu_kY9Rw+VZT)zl0=`Bow$IwxE5pR9| zA`_TRdk2T%K}4MA8{v&o&@@jYM8Oc@g; zfEuw7&Ml`=HV`mYV`6dqE~{MehX4d^9&UG4?Z5&5f%zOgJwzPQdf6^%4yAW~a5XbE z)rGIV%y|mXm-+t3moZk(rvK~9paOjvIxV)+EX+T?%mJ7B9o#sLag;$C1eUvbw2)0h zvE*i&7JP^jWT$H~6U#;ueux`t&uDhF!x?NInjxPqIn7SlTHY^T##XCrnF|8w%W#dy z(&Qx>0DYO>zrG9?^v(mqUtb1kdQgQn<#FMJLpqL0n&X6tkC2@;tdWcEC^Uh}%;Fsg ztYh&gr@nLf(HnvCB{;^ZLo!PdwwP&nZj{)xiCSuUk@@%6ddmFkVeD8riXJ_Ff^4K; zXf&o_TD2O!jL91~QeV;N8tdaNH6+=hWf(;3Y(XMUerHc4r#JlI(%?(Tgt(_sJ8=vT ziZed+pT5ixAElBHVwCcy4B^xqbDgLLu|AFcq6V7B7TB)G{VX_h(5*nh@pNU`F?+Qy z*3fw5f@0b!;pUjv*x?L0fm^XUaVSoxRO;lV?9~5iQPXE?q>#kHmOvk=smG5t?phT(tj%YgDepuWswXLcwifgz|b zllZSMGaf=k^(wasACkfe1@Ol9g}xfp?&9W_LRn_v;A&L77d2z~Zt11a@hyKcY>25O z26{-}d?th{O&?Wxj4<&|2?cR|*SZ$5u9rgvd_P#V5P#)fw?w7zaIM6WbELeH>9!OB ze0b7w9NN?7SNy_%`ZALL`Z5~->C23R`Z8lXo;rc=aS=^*<)*;Y>Pl3al#lOPUc5QAzy;g%-!;V0@%6SzylXvGv(QiYS3+f zjH~Wbq~q1+YNe_nW847KT}`QhVRL4D)F4iVc+JXHxUt{}Zujd52o3`i#$-~`DHvo6 z@}t0CR}nmG>{#0+@d}z&j|ojyZEWD!03B)G_vjQq%c!XQ@4(Vey+ke$uw>^^ehC7W ztf_|dX66An@Y*)tLBJBu(tiU>q6GS68_Q)U%?|CQ_L+^PGe0R2YL)9}Tmf7ewBQ@m z4MhR~)j>q5>*adyOv-BK@+|Eoe1$qczQ*nh{Wyj%F!=Cck1V1rT<6*f3QgjjX6 zf4K!L0NJK#f4t~H|Hki#5PXtcsgtPkhAA%k(N4KAs#fzItOe5%jhQKB4wO|v35UxH(=F&)tK*kH^dl2< z5%>s{nqDPJU4>`i;?tF&($l`f{{;1AlsXCsEa@4e!F-u-I%5(Yy6;T1nDf#}e`kUo zcmmwV3yY zu~8!JlQ{r5XolvRMmE&sH_`h^)2aL>1PAYV`%PjgWGHQRcuOX^BC7RFertGSlsKew z-@JM)y}k{N$$a5|fI1Fx#iu(#l(R-irw-G4ouSmpar=#deV`mzlglACmR{j7Vo}NhVftwSsYOw%&PZV+!6&P6$^5WsW z>i>IWCUbTF)U!@zdz=5k$1vVk{`Ip-2zab+w-SR4cd)eJk4-feaP*G$o(73uVD3~_ z3jEc;?fsXHFsG-OP1Pr(tN0eHD6O7q)a}t~=K4*Lwi3b-=HyElE=uBU|8M7WR=@pU z;Mi@!#alap+L0?|abT+g#Y%iLr(ksy&0b%%u)BTy>&8;Fgn{dqT{N-GLcouZPk%w6 z9q?v{9XJ5`uJbJg@9MvlCBy$wmZ~#gsBxRyRR##Vf131%Ra6xK=3KU$Lq$VNyp^u_ zmAD0|v{POoa66@RDg8UBmIm~;vk2pT>>dxc>0F;j}XI_>V;8u8)G{En5FZ7%#sC&S&|&u zWVEcfk+gsgMLIRpm|NYYGUMWA5t8HuT`Z=Y)Nr^AY}fB!J@1-MV>*PE$I48!NH`39 z@bQ85*@JcI8TXJt>w9f-$*UyI5^A%|mXy+V7b8^td9NcB8 zZ%Nx&V1BidWCN*q0#`Ftag?2{?dg3@J~y9xPu2troWPEt<&ns2e&H(PcTEPmYJvVi z(w}wXgevrX1hf1QaJ%{g(^Z9bMuq3&jsu!5jcT%PLBuHI@pomU{;9=vuDGJKq@ZJ{ z$jwpm$T6ZvXWjE~^1m!m#)FF+N~3YV5B!n9mSaOVDifEHI~?TG+v#?2Xx?5$+bn~- zS>uX%BV+wrGPGt0$VZtRA~&(g=`&PSnwS@Vf|L(!Fo&C{X+W3=miyrd&jLHDLx zErV0_|VyLS80bSKM zUMUuWeiPi7bV;#fG!hd4hMffNb%H~ ztK_XdLGefD!j*@c0!ZdARV3)aIznL^>y3GGqoY$A-?TI`k1nb{VfC4}j-JO5#PPl0 z;SvOy=m`p^hkRr!YU>MR=~DEOguW%sh%`c2^|POwG#%}DB5ER2yq?DZYxyZRwPIw6r}#9O;}zUBd;XvkP5dPQm-W_E?wx9PTW=8sFbB&rAn6^%z@TGB zgL9-o2tvsY+8p|;8i>Ra&dk?iiRIv&(o(=zC;EGJGp#Gb9h&~sB|83PgI;s6lGZ*0 z-l0KuUDW8!i6oAyAL1fe5Z!M0ERe$mKIWXo z{wXy~F*GGGY5Adv{;ZCba)>XL=3G7)g8uVo=sm?TR-xY6HG`5II5tF}@cVvm~Ne7r(<_RP0Myu}b}RGckJ* zQ`@4i>|&^sjaltecP>mYK4uD{V0AB3vuF*rQ!q1LZZ!(%i+b~j5Pr`mXxMR-WrGA< z+I32)iKpwkLVEuM?>&`s8fFrUEr&38?4 zXn=Rp%M{VWR^oJso{?`MT%Qafq<1k30#9D>htyOS3|X7ZKvyddC7?PgGg%{o3rRtbl6`cxG^r;-=3Dapjhz17ATw6iN#Z&q&2_cBbm^hg){EgQs_$Eufx7&?i@+tPMllm}3aA&z5F6B*5FzicUg1O{?y&a$6@`pO`Q~9l>R@kP=0T_s_pgj{ zu{_mN?ErkiQZ?2lkj)SCeJSeFs(ImWe$7>D!%K+D&7bP=@@FDRlK~ej z_P4KzEMyWkcT;EZO@sx;l%+Z$YSMrKfv{KG z;%q2Q>cCmK`HFmg#$-VV3r%Dg$cHZM-4qRBK@;bv*;k+ZS=#lA9^c>HWHT$}m@CV( z#&AM5hFr6byQc;wO_J`bZ@?WfOScHliC8SfT586U-(@&wj5z%t%W)zKtD9|jj_+nVOtEHr+tP}e?xs1wz| zgr@1N7!cztPWDoW=G|{Kyf5dp`hDShX}dU{-%6Tq_nUhiJ-^79yJqpRL^9vz?nc!l zG>OgSM04xvStxV7%43?AEc4=PLo)jvrMxM{nQpK5s#{iN#F*y*5tII1mDAYq<1R(K zLSFIBRSzrl-~z6Q)q%guj~p+}e%C)q9bx`gB!Bp_sf8CW#2QSn6QQLnv<2n`hsl4K zrJny`mX-=!|BG4j|9>z`v6(QAYu@!O8!YLY_*{0c-2Bu8inyv~J%xJH_uOrznd%O) zr_F3MlAWVTdbcNr^DhQu)zsf8YmwsC7Ykgw4+TagBJ_bYt-q!}FYiCpuYcQ8-@FJ* zGJpJ}`v8;laM}k}g_W?Oa$dEzJM6g~9#*w=n;Q zTbKaM*X_m}k%>Ft!)s60r>FDtJ3DZt$CpG3hEB|SFZ3ua)6U4vI`$akQ83)EJ^wuvTUydk4&YyJki*?Inq= zLY0_05_KsQw0r^_#|FgCgAy>3Q=M_Qgiw_~nkc*;&EFm_%7E{O3!o_&|L`-8Z5_7o zY}s*0RDKUx6oSx|veh+zw8)X42*u-b>P;oR_xQ^r6_u3N-dW}~KyG*W?gW=12dC7I zus&R4qVn+@t^18tjCbU_%bYImx+v4L{_3BrgXP^MW&scU<3HHV9o#=ACb?b*ja_fs zrEbq~GFcpvO%?N1W~rVRs#ur@Sj^zzUS&Skg+GXrDzZLUPJLy+Tqmt}nHr>9^bGvk zpNc`(T{d`&`lE7ZhZv&+LXhPV#g1rhQIXhS>!u(o%Dy#r0Wk8 z{OeY#_HD$!3Nj&bU6a0unoQ-U`=)z76t&Nagx-B?FT=6z9&)KVJGOIp{L**%iL}&& zQS;gp=F?*%L4e&sVES*<{fm5~9Lj`-N1H`%ah)}pUZ9`mDU~GG0Q6Ytjp~=A+zq@iN*j^9caQg@i;uEv^2xOv z*qpDJ3)$S{(!x{vsUrbOrr@?ZK^thah>zM?!~Rfr^i0?Y_fM z)JbViQJ0R~{B=T?xO2Z)(Nx)jpb0{!IEp60&IY|jv12O(2fH|OP7lRlL0A(fY_84% zeU@iOHKo9QvTk*|iC}#>wMwu%2{&@(5r%tYxelptLK$Px?mYwhbZiU<1ry;v&-J_!l6-2=}&@hr(9 z9sK927T(~14g#h|N8()DW5C;x0Oq=9Qic=veV^MUaYI5hCyKqL;P7qHUsceRiqeQI zYt!56HwOb|W{0z~9}^pcN|gnIh1pE;g_|DKICM zoOZZ}w1~Nn6@P2slhx3S;tV>xuK~D-1y#A_VCw+qNxHopN&>Vo?JnNfR1ImU-*#HX zmoIlG;3X?@5lgFR5c5PA@8~P|E%lBB1;c44LH*!!qPpIoJ)WsfGt+P_eJ0N_kIWDL ziTS@v-?=VMZQd3^6(Dvbz9MK!`J`=oshDR2fdyM^Ds%8Z8V3MuvB%_$)WveF4l&=) zano=V5%uW=4GbJAK8@#7>x0I|o7-ovn;C}#;hi{9SwYpkEyQT+L~Q5C_mykn<-@)K zE7J8a6gK5`27FX2H!KHUwmOdfiI8ZX%8-G zp%R2FHU2}E=3F$(JuFkXNkQ|~brL%6t^H#XAIS@ioJ9 z{q9{TmN0OgR$C9ycG-!eQ8NI{*c|t0Q`^ngsq5u6Q;2{5k z;ZhSaHaNje0B+R7!`CC1pZ}At{kzAz>o_nM`!=lU-UyUtwNn{fi%Cpgamv^~mi+5U zkQPTO&2<9m;JkJ18#S;+TR;6qeL~1zngOn57q1BS-j>a+|6`VXB0j8qTJE=f{BM2$ zw;CQy!xP#s$M4Qghx$BN!xZK`wKRD025>OP-ofI#5cbP!+a--^4FoNf(Ob3I|AUt7rPOTyK}*n$ z=3i)=L9h3r*dIWYVxZ~%TD7^01og=cn^pEJuK%Nwqs#;wL1lxPl4WuMz*>${);xFU z2F#{^6%1KxosaIpE$=sbW#GxR;FFL4%=MRo_MBL{*J5lYdCS!25mQ=D8q#YHurR6j z$`TWJTqgYSFNy=$Dt4@csCJm)2&U!m2Mj1OWX0+L86AqWj0FfLyrQHluJ*yGw3MJ0 zAG%~o)O4ow{0w_dTh!W{Lng|N24z3hqPd9ETZYUo@`EiN&P zUAnm)(EG#k3uV39*N$0Lr@jCW(NJPVqN<&GL~m^F(a z;(z2U)Js&3*|38ST<*wFZ5iO<&br~WZ|XHPh=aWn)af$L!}Tpznp{rInBlRTuLP?{4*xLryJj5)5EtpkgdckWwN& zCBGg=S!17@t!463@9#tvCvP|8kW;1XK)5{rE_!@bB0gi!wBiiw@#0}jQFAC(tH6fj z^#$FmmiohXP+3A`E7N+4t;i?!0<}Pv3oXk*3;Hybf{0|BNUS975^2$m$$W8KXwgDbwox+YH8?@DivyKyDpt@+>z}7m32RyTQ z-@cmyy*~VHWFn$2hURp?yu}x=Q4iTTwg{c^0)m%N2FHPT>Ryr_m@Aq@@O=VC9QfjA zzicad(2gEC+PY(M_N+Tv8y%KR7V1)a6p>ScLOw=b=^>d6U#rY$+^C#vnCyWLSO^7w z{lmgnlSyppYY0w`(wV{q9Y))*LAy2??5Rgd3|mo-i^TZoLMJ9sJw5F)AWbv}Pp`PU zbt$iEO4ZRygSXckA!pvs)QCFjmsj586wI+jf|MvadpAwo(4Ezd{zpeq9~G$L?{URzchmsA|$mk9cK-*4`lI>G%)_~jZ^ zJqesW=KONQIDcpll3&4iT__hbb5_p&>|^?-#l@%ZScj(N?nfQiq|eUn&#uvl$hUW< zTC&V6^2L&-KZ^#{!=~AXiWx(PPmVyXiUpP=L5j_DMP&twh_N_8mK=I`^byzE=S3O&=TVY=M9^ve%qLCu zc!UZg%8EKQRJImvNe1{N&Pd5l(zxqIjo5IIj1-n&~YsRAu1JzUS+hxy{q~?$pHl0An614T0Onoyr$H; zn;cgd*n*ka-h@?YN?wL=1Yc`63LeFj!JaVt>vQ zqbL+8BhinSI&;NuE~_%`1v15=WLptq)T$w_Uc>HJ7*pLudS^sndbN0QN)r#+a$`!1 z#^Sde_a)%>%@fxxjW$rlcZ&YSU!daPivuHG(PRF zEDk)4;VfFLp_^>K+HkTb{|Wel@w6o3R4-b>ZFNpiQvBKU*3)P>bOM-44U)Mj^(vSb z_n}TReJ2czSG4)0d!Bc8e_uA)WB^KKJr0>a@581S^r1i*_OnO5C`af+=~}y`YW&;@ zg*TlLycE0(u1lk2zitQyZrZp3{2%$Rv>eDRv#u7R`E{P9w?mt@OXnngho!!XVhhX` z1PEmPbW;0cNA(5$$ts50GM+fw!w9qz!|b{FeHwLLvb=@6hZr2VHYX|{)dDhNMRkL@ zY%+Y}f#q7iwRV(ZTJ+S-4va4;#3?tpNV$&0tkOnOL_ zbfeQ>AJi8R#~pRvoH0UTv$5b|XSoR0Ng`nD<|*3pxWzC`U>H@R8Bf~()OSvYdFOQ) zxY7bwqat(&QKjbg6G5x8FRsZz^2crG!nAfZwxj26=#?U2%aCO#IWy?u;)bvoh`b%+ zt1Lw)9-&WPs~YC1_*ozt+Uv>-HQz?wD8V4Ar@nC=2=}Kq7X2of+48;yuL_%gXY_R= zGR&_y#&KdVLdIq7KQAd4O2f0Qlb)NL)6Dw__ABa&dK|>X~(-X?>bzq$-6kC zg_!{JJ^~W#g=KA75cu-}-Rmy<|FV{(n+o8Ct4Llf)|LL7s;RAh@p@sb|9-T7mz8or zrBswPndX$lm%ODgY+-e^2BK=9k`so^q7KOEe7=JW$?G@Pr*!*5LMJNGp4BK$x48eN zYQSQpo42$N0#l0Kd@d}f)IX#m;TMg5gMm?^u6&_p*-|(i-hVdYa~u7 zv!hTHr*kEwIhKmHQ#dr_V7E1&VH?@A`EyY|e+7%P|F;qMcPXX5*cK>2Y$>K1dl#cc z4Q+(j&9ZNf8tCH5h3Z63C(py^v5#xeN|JF5%(=o=uqG34tB zwW8`%gnA@9bx*OEPSoL|l=hXYYnWousyWdt#C-H*HYGPfFfT95nphVmsQP`JbAqiB zyt*kmRgSVBiVZ!`7!eDG&-a{Rv3}3ZY8sNP>gZ7pKMi@wF}s)^Q;&oq4^wxu&qoJU zbee)=e3LrRuN~S0DJ#6ZLef?wgUJcc&^v}O-jaclc zwI>NVdX06K5#6G8(o-3kq4Z#cF~u|mSX+l$kf2Av583eDT~%HAQ;!A^Vnxkmnu@X@+g&DM+>2BHd%FT92M5ZTLp zTR@ZfWfQoJ-u+3S*9(kLj!@3JHA0QgLm~MK^z-%Ll0z!PiF-a0nq>w4f_!S*&)28E3x8h}}Gb4+Ip=$J< zj@H_%{6aqFw{Y)tvAib2UN>>NV-Hq*yf)3SweNHuF)^_qK1t6Hb^+hpJ!dpYOgVZ# zfr6QQDd`!Si&8oghIWhkzERr6#D}mQigp*?#D^$lH za+UDy!*A(QUD@aWZbY>0U-}jk0@~an_~B;#`kv5E?hLOfX7}H^O??k86GUR#a;)%C z@DeU;)k(2s8)0v12{MuA#~zKot>W_R6fb#wKU>Jhw6a# zm$did8^_BmO_Oj{p4N!aNE-wP7hlP&UdAh;_VUW);{kQRnhj8AhFa6Xp0l~uP#Zrx zL2G|=w`qjc-fOu(S&T7>>kp6Z2K#VI*;ZAE9y>*mc}EO&Pm^SV9r8YcLw*prUGtvt zD#|AI*Y~HeBpO&4MmVP#YgsFS_epTapCgvCSb`};0<`U;cI;H_8O6wD?n-#B*F`1A z`hAz7CwTf(BXHJR2%DO1&GZMYjQV>VznH)OeH3kb1oJWyP?pD7d2L~<8^_qbage>S zn`XR^ZkYFG2;7Mtk!CFzYQyB1p(3M9+%wEob*1Xd>$S@;URq4zA<&RVL=0hoQvYo; z_|>^=SO6Iu(lh1$E+iHiUaO;b`DFE7Ojyk=;#`da*7+j)R9f|{tQ@v|LVQMZPo`Z(=o$=vWw zZ)XbYz?!*h0tO0-f3q4y^RrVoIU)}J1NUj$IKD?bTm;apMp#%Y_QNy1vAvsHP{Kt( z=Be}~=z#ifr~{8l)8p_pjAkkO+lc~`aAMbS>&Zi7V|>Oi5)lkp%T3l!1pR%Qd#1_~ z3@h>vs-$0$4~$6~*1{Dj6L|!@(}^pEx)pWA?K9{d*l2Fto>WqdW0T2MPGh7{LA5)x zHR0+Zoi#ZS&4lz=g|e>3%ub9x;O13Nc1*aWVTBtCU`*hpUK2}>^#2kwhAayG8oUZf z#tLQ)<&)Ka!cA6xaa(Ia7#!SEyrRovW*~Lp-+n=ePHEm}Te)wO+fmlxFgyS-p?@2L zrPTL?f0KNqFVuT=3Q2ApeZ?e85NHAPaiG9o(oETpf_@FQVV7RKJgJh5CEdouOb@um z&}1~rziv~|j5R^6Xu;5o)%G%cg^%y~8inZiNm83n@KBJaiCLy27$;f(#T0?A`>0M_ z{GgnfusdNYmP<0hX-o<3;Wk;Ci%8l%lFypWC|8NHcne-?zm=xC2$#~88!lpZUnGs6 ztFLO11nr1#zj5){OsbjVF&OteqF zmp*eC(79;Q;4X~M<>H08wDTeFxdw>bqT~M{+1%FD-_Veu?(S{t_uZY6WW{7|&>$f1 z?h+;yApH?qP2FwI9`P);n8@pOoC$vvoUXF5(*6@_L&Q9@L`EsH)ZYW1X)^rb6ddkO z`qTad$SqnRtgNG3pA@0dFpk~b4}3c~smBLCp6-wX7e0JFzU2_2-hCfM6!C-+l*TCc zE+8+PQPWbVckJEw18O3d^FI9T>mj~dp^Cn@Q<`Sus=-qT*jC@RMJz^J5cb@vZWD zk~5fv0rU$>ZP!;z`C2BIT(W!JuNuNbmkbBK4uXglbAX!WmE1!>S--`%H3{wWMz^mK5u7-+!2R$WQFk2ao1N5E&I+0 zMMQ@gB~Qk^yNmWQ>;vbO#^CcaZ`3;75;?q^WbT++ z$=OW*-o)=%kBEf`p@Uy0eplyspeR@1cXfIzvcfqENEcd`9wu>~b6D*du-)x^;)By@88-!KLKId+}Os?+^?>I$*o_ryz$yK_J7a&w;8%x&FAc{ zY%(@zeqXb-$9brdY}jpXHmh`)&4o6bm&cWKr6v84Z*4~miw(3L)B=GV+?DHk%pqqN zD*M~RSnWf7U3~#(eA=wgz0YNnrQ4dNeWFQQdT&62b}SN+XX@jDS?ue+d?BCp2cRH7 zfzUxfS2}97v_rP`9zT4#wvUA^kUIX$FIUq)YOL}s=LR%70u-9TI!?n31Pf7d0%AA2 z_vA-EHVLmCsS)AW#!6g=ba{#8zsuA;Nwj!!;vb)KR@pye&XfBG_PcT6)_YVM+*ibHttc1e>lrQ>t$n=e49mdKkf~g%j z)tWSS!Yc?z(2%}9sx(@%<1F^irD5SqD)P5m8_8OaVzf?1*{tJU9+Lrw)hQxFUgJ{c zPv3lfa+p1`Rnc_&wr|_jQ3(i|7~k|&J8o&jqdJ!-Lbq3{{EJb92JVZ8U8xe%`Js!+ z6-z%&?hlXo>H@!XJu@4BUrNpkoJ_KpfO(~#iTUKAkXFtY@;^i8*!?kVU(t~?_ngjT zkaNT*U?Do1p6MC0%S4Dc%F^q+zV18T0XNE$tXk~3gv(If@(f|<-S|xVlIm@#Sak|t z!V|LD5cu9Emz`mp)!;MOT*4)FELxgvz zB7^@pO-&?5z8>dYhma$66IST#Z#H6G9nFfKtFvyWGu=2PLi-Rxr4Pk$ZW3CsHtsZ( z^-Ih2?RYzCCz+;8f`*nA3gYHvwu$p3yMgj_H9K5=(Q5eytbE@6a`AJ$>v-9*sy>>f z2ij^1#AQ~t>o0vht(s}yFn7bI#sm~^QGN<=YM0&?mwvHN$c={OmVY%6ha=?P}!$gz*rpI0BZU7w-DK zCrJ{>GTtet4TaW|`Odvtw!z9VCMO;ai+hq6j6PSkg@E3#wOV%D4eNwss25j~#dpVn z!!GpeABPX(PWWR*DApxGycmY98p=z3#Gxf=d8-2Vf>v@J?x4C1QklhJ7lFO=61y$` zqHOXzpOt8(zdp}$sRtUYaC(|2e}_Zox5z*v8cAnp3rEF244koBpt=l%OYsk{Xc^o# zng*Br*~OCQg$bdmBB5jYljru=nt*Ut@W&EvH9KrJlcw3kT~S5da!6t0%m-Vr+L?xLlf2)<5JQOUV z49Wg1xP*-oM4G!E+Jr&} zZy~G~=8DnTHS-XoE#}H*`ziwaCQJC#MOiDm#GY|Pz=N?`tteB5^i-b2z;ZfioHFq4 zD5S@-xL>tG12n2>mc~=G@>0kX@M>)Kr9J&xMXdJ6m>;dS&uw9=dY-8->g;N7>-{Hl zT$?4Xmw$KC`DIj`E+b!`5^`CT#RQHY71I(RSm=}ecxw3 z4Up%AsGSGq3ECY2`MvK7fLk-i*(U)ApDSVy4$9_E;1z9RmdPjuB;j836nox&pRBI{NzPq#RrYIDW@Ak(|0bU8iiF@E{rkoywY>kz2#W?LAbxud)F zpG{4p1+?uHOEaHk&1_=tRk*fWZN=0r^pv+<0ivdX(C0TXJ;rsfiYs?lckJcP{@Ked zG=9cqOJoyT%6Gn~iuCV)mGP_gIiy1n~th$ z+HE0A7NFLCx7P?0s{SmN{wjTXLVfEu5745)nbA{vr|K%QF&%M@v?)!A$s z_YWhVySC=H5fSRj5)=RDu7oGDo=_cvc2qr_HQ>K=AzDvV@c!*qT?^ zNW}#2TKTfjls2(GJ~D8e9pG343<8v3Bbw(ukumI}Fzz6HIPvn@_JX!t&tu~6Wj-$# zqvB^p6h8D5qlQ_>>G#3U6PqDIx6FN6tZ}H@*N>xM-v^1?1%-kIvRd9js~RZKs^)`b zs?KQT#g}VWFLqSkgE@Qnc1+K!)49r=gPcRR@Zx*j_zc4+KTC+wbpmj{ph{DMs7TX? zK`Q&46`$he9^7ltU!FJIKFwj7ZruzxD`{-aEjMXXxM0!y)i0JFgWSu^13d{75UF%< zMM+CTf62-~&;0GHRdV;@;hD~|SVm{6*_%}1DT&b2O-I)|>yN_ez>P8B@3{hIV7}mU zz}4YB_W_W`|AY7qVwrwsAfJxbf9rbN$?Ki{ovE3(P*(08Rh@?0C>Mfh8b4zC=L3S< zKwB312(42Kwg#eZYSHV#g@mQRatA{xfO%ZNlnG~ktqyUT8u+<<-?#gTTE>)1jjLnW z`1q#X0?Ze390m#q5ReF7fc~H_5-@c&ss|@Mc$=)h@#fRgn_Dnjf0t#&Llw;INHX^Qa|-amz)g*B>J)n0B0Ieff_~%tgm~* zXUhJmkBZ{QTo@RrU~7N;N9z!sgayjd|M+W94ffr0blm&}wzY|WhNQ^~9Uoktf7wKg z<^b|XGq5U{O1)`!PXe#Pv;@Y@mVrmvKCv_5Z@`DC+w=VG(54^BLTT5)f#Ti$f#FY5 zA)h(WGm|U3 zX$GYuNkLZbgW~0iRCGD9rd1_1=JDnv)?@s_7OdSL-4S_DI2x)-LOigoDcT(;RY?t> zpqhE;uXDz+6(^o0NH>Eq>!J1L-`12_rz>RYX(?c{8;4ZM^uLG5=2(%vWUFkVQOWse zNQqaz9<$y2-DLfKD?$x^$qky-EYX8b*P-$fSop?Odri-bNfV6{s|6gq%|oqq2Y{%Nf^5oL%CWM3Q0sT%1b zk}+xEWzhNd8(e<)OmX1{@^UBU?Z9eDVen!tXj4-d%J3Z_kzD=&4N)@~THJCzcTRrN zdW=`QYlK1j;m45kP~6yY>f^t>jB8_hC{v2f)>u37yT#^xH|lafHlgZxZqk?_F=3)~ z%WO8lfiZnr+OBvE>t?B_?;8V4%^am#I-H=BRm8h3$KEjyunu%gu>d^svchI&ymd=z z@fRW^Rldt8XGG>poIylVeH`U*A-VMjSuu5 zN4rGArD=aB%I`)Nyw1R6s8zjh{z~@tc9+Cu#MAuYCnJVnpPv$=+~UP(GwO2?`X>?A zd|D?GhvTo_?;->(#oyzyhSMrA&ud8q;gM51-q>m%cx!<^e!I!-l!f-X};ey)J?A%_cEn!xKxkZY0M>yK(qK zfp`q<%paQOh%V^d+N2|sRQQ>*jK7s!h7p1iID#PukEADt3B467Y0VR7xSw!dx);-J zR~R(fOv$7tzv$D;7=WKDdJMe;ZKG37r?Gzp3_yZ#d?k} zLVwcEWtda4q}+4#nd3tpnRt|-NV4WjtD9e$8;ao=SVP7uBq+{?p=~&8U&{t_b`Xug z#*!1EX-b-~V&%+tC!Jx^;#O01XGy28JA$PUCqUT5-NlIW+|e)0OWDi&3VI{YE3!kb z*TibW#9$mdBwCri1X*jhrd1KuOv8?1cYx1-t0f+}>xWCE!$_86dZkySwR~dyKx75L z(+REzl*{yU8WQHW0A@R;3qrMN70Eq_Ce-&?)M6^7UIj}jb2!O)O&tzSbKpf+6_}1Q zG`M8#)QfvsEvFw3tlOHDh@A2utS++Jr6U=fRZF71gPNi&*_qAaMg|$zIIDF8mSSF^ zY_UgD0Bquw^}M*`)c94Qqbany>I13!XrTq4l{A=5v5?@J5Pykp)9*BYd^wPC7jYc5 z8!qMuf;~P6?|zA$j!H=+ybg!j6F&fkyc($(S98a)caS^VpX=AjSqWO9?SQrD;Up7Q z2nEs5uGi;mR$?n8=i+mtxqA%e-DM3kn$L4fT6KRDQ%S|2NLk#T&^8s0>233V{cX(= zTr@B;V;JKiQtaz9#w0BaY^H?SImT!u#g6Y z#?6y_WX~Jx+(jOv9zV0GVmvaVfFizQZ9(@X0S*OD9I8bHIMSkFlmy8Bm5L}QKH@6_ zmywZZwz={$2sT#INLJF`{y50a0d0!1h+pomH*3CK4j}6`o@1EOP~NMd(VRoz-`l?i z#3fcg-&F|N$1()gF3i{Ts=(4jhfuax zJcY99N!jmm9UK9gTk6D6S!b*|7vQEkIkOO|oZXrtQaxs#Bw?ExD%R5PD0(upzt80%g0nNuP|gmJ@Sp@;YYGd5O_Wt7;MIHKM{#zuBj z5!lp_DF5(1-!R2Z1;aPS_mw1fuHPDrdxet#TE8dgjaunUPZ&0D-7l9YfB$23te`$$ z>kh><>Y825eAEU&%3~}fkRU0=FtoUj+P}seKQQb?hph!DIz3t#3;5?mJ1eTdKDxe4 z%%zZ8h689O3NBd8L{LprLnbAzwB-&>+E5RV^P~@8ml92KoLi?SC;eU$}}Re zTJ>IJ!(Ylwlt|U*1;n9bvMO!#lt$}BpZY&__A?t0^$mPReZ&}OTIw~LQD2=Ug%b&8 z?v!ULOY;NmjMkt?AkVZh)sQqs{=9X5O5MWk;u_*}0Jbjd(m(@)0&}{$bKBv8>&cb# z;r=%ZFpugY>N-@J0IrM(p6%rhXmHO+gDKf`ugsy|4UM!lNr)745*)n zo#W8c$OEazi6NrAf`yS)`A1|}z(Y;46gv8IX|;+cZUr;V(YmCRh|iyE!F%5PN7M>oiIeB+zE+7EeF{C`6?}mh280b z1TJjWv`-H;NY{QDEbEsd;L)dOD5=FY+k)X6O(a8L6(xUEj7v)T4$6@qF>}If_GIu= z2?^v#qLk7pG+>`!NeJRj71F0>;?KmNobrg0WeJUwuiN~FJ!CjG3p^xkGI>ec3yI?Q zvtq2xW}j*y(L4;iXt_}E#yo|nOG%>Soi$P`F(|J8Ax`1B$+$_%AaIs7AElSK+R}OI zD%v;w!4nd66EH`*9X=@lOH@92CrwKkgsCY54 z;$_<*kf2Ka(Tz}2*a+Wlw*u>>A z>US4;Q*KK66_Jb(g+n%(nP`law25vLqs6ft!7W9fBkWo!xc><)jS~>OGFBqaPR{Piqysgl{^8-Afj1T-xe1z=`vGT!?~L{KjpCkB zFil}x=cF?Vu8>MmjitQA?;Fr3%cJWyj}GB7Me4^n@}P<^)7izXSRfr8bQ*%)D5flk zXE^6+@*wNm12YuyN50d*g_t7Yf-^haFikd{SnCZzQxtAqD4=D*tefVQGhiEH55)qU+nG2@S4?^@d6CtE%O-6OAXg02`l7OU~CJ%W3^2rV^`*4;~<2)Jn>< z9e(uA*V@u)82(W)$e+iD2k!Gd*Fc}b5^tV41iMIaSgMB*b(Y(Bib(HAnm2r!`fm-S z(r@@L0eE1rL7MU@dgve6eb-32z!!gF-k5{6OzMb#wP1CPv?B)%|AWBfupRIt=J5E2 zeGMK6VOdEEw>*PHnhGra@)e9Fw{t-r_$Y*+aE?f@&0{I3r`7j8Aa+Qo_#P2f-1Ru4 z56{;|@hjZ)8YZB_`$xT{-;2f2J*Ep(D3#TlWab=qT7tlZlzNGgcxPRZz%iD51u1n;JWY8i z6eUD_=kv4jj`ABAn4mvjxMJxwslx8vbUGfDc=(Lcu?lZFz1>_~9!xwQ_J?n7h9d_B z(Pb%%A8rHq{BiJkGwV(Ud(Ny+wirFwea=1UT*wC;i=(QQu)CXggChv2YU9auScxgy z$wK8!(I-)E-^~`U#B}XXKI{p%V_NT{?TEH#@UMsjp5M2H^CrE!BL!lw?;~#GTZO+d zZ9C^d1Q7G}+w-%;DimLfx`58gN+BXVh}z%CPoO^Pba{5*@CkPbjt zv&Cj{W0~fhw1Uai;~>4eMSCzUaTD>$H^f8qwHGhG0})S*jlr34k_`6m{T6-=yL(fan^U-PtA0?im(Of@0 z*WqPoY9pi*otwNMhbQR5Ik3D!EdwgR)Y3~}gt%R?ur^JitCi(1-e zuoc)}%UL_cFe3G*91O%jK-e9Cu_HIelNb;s65;JCHXi@>buBjpsVYmaTd2^{(IQkJ z%RrT){OQZaxo3ey*i@XEbQUx7j3WANvM{h-2=-G=^%ZW}FWEG&U&3SDV;IR{NuMHX zoI+nxw81k&s0DpfT#0mDGu$`r@Q9Vpcn%|~zrzg7-NrZP|;a(XyofXiqe z!xhKrPk(qGc#BxCby-+gK>x}FeO!6zES2NfMcfK!CFk*?u?{6KS3gX( z87KX`+PjOCGv=%sJmI1&$0)<`xXL&X851O_i{&;(RmhYY_{+2k-3v^TEZwp9l-eOW zbE>f{K`JZwAyMKR#X_xwTK;>H61}(8E{C{Ys8w-~+b*B@p_;(d(q=qXHQ7)3njWA( zGGqrxxD`k4$9anr++0ABLJ;}*uja1m3#hp2?b>=BAM_BrpWtn)&BGk!fVJs5<)CX=I&$D^AIHp3W)h+g`of{H*2zy|iSr%pN%~`nSWV z@@_JE5r4!Nw%FVVv9T|p3-$oHiRVV;3FaN}mVn23{XVW-(NS&`SkW69brE_|V@pJM zA+ZpeS95PQsFQKAeuWK4QD1)!h!4U1JG)LaN$hlJ5f=`-ijGJ)a_@&ZU$Dr&@8)28 z{T(Q%L6cMqPe*B;2|sXbTf9KSRkR2u-%bukX zFr7rD@Qi|ZrRp_Pui|-uD`W5aJ-{0~qRtW-^o8_q$?!!Drcuz>DZ>}1nslKG;Trw2 zKnyuw7Rjf3-^I3Hjsk5LAY3o6S`je^rYjTiI3CT?&n-;WnL17go9E$$-y;b#e@8J1 z4q^&1%>SLdKk1S`q?@iILnH2Pgg&x&%n~;F4n8byvg_&KukC27PMm!6cN^|Lm>siQ zrfEpcGU(>&2KPlMMJveiGxK(&F0H%%bn1g8951J*I( zE|kI$jS;>!=6tRM`F`-2TJ}p3s12{z>;gV^D8xgjUut7vW~+#Wf{Ln?9reKB{pP98WL(y}1u-B{5D`va$>?o9^N|H4w?+9BGG=5YO4G6lcQm%lJ57;&XR%eTbm|hp?psBAG19E**qs2 zJlt+N#_Eo)z0)LqaQ^&f$l-&E@FsQ)MPsb-l3Ju>$2&j6S!tosx{fu46jzMOO-px} z`Jf{>FI)-i`4af7P zj|bNtd7Ad*9o>C0V6Zg6?2q==rv~wYbIrhhuy;RMF4c-Jll>+{)5kW&V|lYRL+*V5 z#xS(!eh1t#q>#iq%#S4P4)2vTn{@mqzBW~Pm`=n-t#(iQ=}NH*aC3Em`LhmUtSr86_OdL9~DSQ=w+qXy9j3q@ebQTAr!I4c)V>bGY4#! z(B#4yU0vRFH~rYd=u2B#&cers z=2x>s)n0@1mJP_3e~(liAm9B-j^+=o=DDiUmL~$_WB(7CgEo^a|Bs93WvdeA?>=7- z?C?|EdWmSnyS_iB&mUh%3q=Y|#T>M{wQe<1rTCBf)MU{WM|Z;5wswt2EuC+f&RG3u zH~%pk_^7pNaBLOTzO}ig+iETkJTYo4(BjB&bO4h1Ri?>5;kjmSvBoC%ErIU?eY>z7 zdObZVN*?mfeA_9|gy0+@Po-C;u5=}S6-&M3s-`+w8jN7D!1q+91F)0+ zn5a{UJIpgrTnhR9yLiYBy{_m_6I=3FwpnB4^ZxqBos*k5Bc}N&htJt3sf{noi?CU7 zYhc-{&CveSa^}YRDp`Z3!-ka&{_j%3s)N=#(-Pb142bEhIFh72G4=zfy%{_pCcwH+ zMGVt(OK@4Y48xKf^6K5Qe`Ot;Clw` z4xTj2`UT}ZLRYxxFDu&ywG(S5gm1~_g#AT!ux#2qT9dHEMb;wor@Hs-7e73pCzoG; zR|)kFyEcY$r89{kr*s*faOV5QcsaVoY2KW%`?v3z4_AXXvjc2;AApggohEH?sF_2R zG+A4EJ;dP)FkfngexC0&Nd7FJ^xTa&EV;0t~DbAL{r z7qv5|X0^tl8B*_q$CIA~~v{m>fRew?$Huf!nAxC1+sa zErVtU$F#G@1SpQ-=QY-#CL2bauFqam^J|hbN}Mivq1bf=gNlarEmYTo$*uey&Er&t z+HW)oUBpK7uJ~*0$ZDS=@-EgMaV0~dnVIeMyg40SrMI)eP&63k0m;_=oK1Kd}&To_i@~5ee_UhR`P)o1Yen4*A;di&ikC7(aH2ExSir>GbZz zzjR#NaN0Z~_k!~vLhzj?`A7L|9-jZas(12gU$U|q}O(`=f*!)04uBL`6CGV{z&z;3)yll2rX!fkXnF!WA za$t7WwD?UM)AYTBD^lBWW*e+V*0rKs7S)C+E5MZyMTX*K`Zkb8su2~R zVuHHAY{i+phA2jrzQMM=pSNu_ZTWvznMBWweS8^DvpT5*acTPnRI9{eEt~mkcWt^( z0#d689N}vpC}hx({WNY9O2+?KWfaZpBlNJmrr}B6gZr~s{!A9Q*;*pQ+p^SbAy;CS z9tA(9@e&C#zKUmc`!rEFFF#uKf-3;iQqLyEy(vn2-)eh(KOP@+YnOYBo7ocxSGOX? zj%LB550skzO`e5rGbf1R&q+Wyl6Bcs=2ZA|!%EWe)_Wd%&m=kh@pyp8mYbY#XIOFc~Gv&st+eV$~Z?9^a|EBF*`Gr?lDwZ7T zTxwdU$F#(1QgBd?hQE1?wN?l&Lf}@Z0dbDh&(Oj5l(oS`mkHUIr3~g-C_bE@Xo{RtPlw^+xinya&Olz*{d@NV>uGs$^zL#vDUDWqRkrFQ9$AJyf)iq zhksQyT%rDCex7ZAVVT5QhQ6c6DNB=Wg*uD}8G~|dCKOIgwJNVoMf*xu#2>U89njMQ z#@te?<7ah0^3ihAp1;IHL9jx3^mSqS7bq(f!Wf@22(u@Y(vU+s>E1hwHFe^p*{|7Ln}@ z9IO_xq9qzbhbh~i?B=lbWMq9#^55i!0R^jt{I;x`F=gvF8kQ1JbsrpeMkjn#T`}@! z6(UDukQk#?<3tgNhC z287sMhpcMWvsuG(WADu>%4B1j>K!TO#EgZi!vY301Su*8YbrLWI9CYVTF0N4|K=v~ zHEb)1IwXIv`z-r>IzGIN-w1yGZ(?b2aIQJ`SJS{E=aOLK(*PxdCL-*PpYlvuK!SBx zCSVV%7qog7hP6iJFLNb#_wTxLqY=cz^)T0aa+0SMBU6Ho?05@<7A6D7-6=4qrf<8P z)eKO#4RFaI;>K!yG$);Kd5}1v#P25Sk;l267~1!o49_)DXOXS)$V+JDZLo}1aHSOP z=1}XPZ#cvjAzP97|Q{8xp7mfp^w6fu=I~FUF4UrH^4M}R_0EoDzN8V{8 z`%sgMZ_4La0A!Ky;jrkB+HT}~sT1r)bh3Dmpb;O2*WRDTS#woAX8X`C(AOr7UoShJ zgQ^`3X! zjn7j182!&T@~q+3bK}`s^aDBuk+F%ugX} zs)1V%B;ISf^P(&t8$=tbdZ&4Hpz(X8r2_1)74?ML9h(Z66>QD8fd(10hDNv+XgtfMzHzyYtb#w7zx4qm(VC8yBO zzPvKA!I=4;6zpcI=&`OF;TmSjS|hZ^^;#y~M-P1qK`p;<94#@6>UJXVTH=X~JYHzO zrj`z?1wvzb58~EZY7Z4JvEy>bI=tIQ0&tPTisJ=vh#Jgl_8BAAb<5ddgd^lX07Y)H zMC#vxD*4J`2%03J8g;0&Sj>r6QF=yV(j)!rylVXgqckBjZZdqeRWRybF{{LGGLw9n zKXN0Ny9@_8C%$zUHLGDtS4lEgXRBd66~WsgHrnkjd|J4&B>}Y<@5M@vmeB<99U`Zm z+}wxii75Q7ul}qENVy;Ev)q*yn&G*LRgd@JuO11bBD$7i;XZl>VHM&j;rC(Bg*=fO zinl!j-j9(yh>L{3x;!3l7F=dD#H83JYc-;zMe2qM4qgszPd!Jo6da>7N|(Ubc2ZM| zUK{c+uZ2;S4E`0F_*6`5OA2q3Qar@)5IN6CHO1BMz+tGiS;48v-8qFYQwC&7C4fq3 zzoGD4bydXl^H8dNd1XfSCo;mM$;92zft{!sh8=TIl}T_p1u9321DAK#ENpJHDh&JG&}(Q zlm%O|dPHvsG{@pLS@LF8Qo zpi4L>nd3FY%crNX8EO9)S&FlZz@D?S9bz^};;@h8L7PMp8Ek0QIkIr&6Q~*c5m(}L z$&rF77A6v(reZWNZ^^~fseh*LAt)noHvb_kikfq~{!%3xG~)^l9OANst3S}l!Eui_ zQej=Vro!LL`reOXO)7sP13O@5#5r13AgPs!bJ@)5Ttgw=neDpoagWYnyfVZqrSdD1 zM0XMXYhe~?iS5iot~Z`zIK5wPwn(8+$x$K_gsl`j+zv}Mt^pJ&;hE3Cslf{+f_zkG zO;5AMgRMiYyAJD3Y8~h#rl^2$64v`kko=v8gM$5V+TKO2C~DJ#oeY_f?u7W|jLruuJ!$R=csjsgzXv9M6zP!!%l&kO~$1bH$P@$hgpMNB*uvUA>yK=lTQ z{GXU8lH)=$}%RQXRiqhfI{Fgj0N^Z|YIKlH=iZiX|Aj8L_|u8E8!`@#zP4 zuZPb=0Vg6hykeB7v5_@b40V2TwoidP?2pE#!iH1lugcPE%2#D+wSDH6@#PM9s2(r% zj3WfL^z;w%JK%X#3?ur-QbH`Hu}?1(7q?j@)*d$6w*(m_FX9r6RM?r6aCG`*>+!h? zB2k7^G3d=yV^4Ze2j_AN;{2};k)^USC@S>gZIiNrCJ@$lP1_C5qRk$upebeIqBhh{ zBFZDm-75Wc|DVe3M2DkzF1y~v#+q1~g1-RIU77dX~h2K(+T(hLvBEvp#R zvdY3Yz{rNnmkzQHdhFa_>#ct1CoPGuMQK!Zq`DE+kQr_j)Me>r9!Bbh`Z{kDge50U8$Eb!g>Z={tkrkPOLF7xEOY^NX!k4$q>1$!V59Zmn}d>LN@v{KV5Uxm z$}m!Vej+n zmNWn}kV88$yzT^o&6H`xJF;vRA;>)}xh%BjU(s@tA9NJ`l*F?n^)`-%wsokfB@?=l z+tMvW)ClxdrpAK$Hf}}X{hL;cnRdNPrVQ>=(1c%!G)*&Q&J>9B62N7MmbBH9E)I zFl@D0GGL2;GN6(-{dFOxGebGZN`c4bMFMF=czcL#kHxuHXtv?!FxVWiE<#fy>&#K3 zqdj*a(y46U>1a9gj+C{~hUZH`l@Ls;P+m<0NVoadG7#EGEUHYFxL zAoWjrfyoNmkQR|2uTWBk<}3!Q^jfQzh-s|V84EdutSqF-Cdiy#l+;OAQej8Wt0QI# zyt0`jNmN^CuRL)x3ME~mLK>M*ZVOS2;*29gHhyrK?mE;DB{YDlY#~mOm22?&h^imd zW#y#Kaow1yQpY)WpO)SvQNG>DL~z*tN|;t~7l&InAaHHpyl{x&5kH|K*+Z9Nn>Wi8 zBl2fr$0Z}o2M+${mU+8X4)nb2GHd~ilMtG;nue}-ez|3ui$u;mF&denXrgU7xW*4c zzFJM-QYTSn58sAcwnS^K2kwJD`;ISC4v4zlc`&!jP61YlQwxHbmDm`&opu)6`O`H z<_KHLFVu)6kT;49&|6VwHxEULFDgMYGVdEV3;f}S1Z3oJn-t%go^O>HA8S3tBvu_pv)+MMI*80t$u z1138LZxkJ_0pOx9i@6Os`V!@=hpN&|bKf6kxu=`4P)%14JwIsk3EFX^?UOkPq%=C; zI(RPWM;b?h1j2h%%)qYLHcIZYK)nTYSAULVEae66jMQQcIoqHkdJ6SbvIyA`a5|#y zu`W)fLgKK%g9N%JZbLD6$C+?FaA4Rp{$kWQqnw5!Bdzx;Y+QU`tY!(X{yhmB2lkO! zXvQwj96f}r?>Ow_Zk@>u9Dh!r5K)GlB{;M`CvBAIDtFw2vG1iLO$3V|goNHm6~Yn8 zZPz=JBoio_7M(4BxnR?Vre+6z>HIi>&Q1aU1N?Chc31iZ@*kJvL0AgY6j3y_+TDAc z$2PVx^sQCjrgmRf#)uaO;It3s5KMq*%B$g_;AR%g&;NzThky?$*MD#J6A29L^V7Nc L=@=|Z9PIxApNO3s literal 0 HcmV?d00001 diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/Chart.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/Chart.yaml new file mode 100644 index 0000000000..11419cb2dd --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/release-name: rancher-monitoring-crd +apiVersion: v2 +description: Installs the CRDs for rancher-monitoring. +name: rancher-monitoring-crd +type: application +version: 104.1.2-rc.1+up57.0.3 diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/README.md b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/README.md new file mode 100644 index 0000000000..e0b63e0268 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/README.md @@ -0,0 +1,24 @@ +# rancher-monitoring-crd +A Rancher chart that installs the CRDs used by rancher-monitoring. + +## How does this chart work? + +This chart marshalls all of the CRD files placed in the `crd-manifest` directory into a ConfigMap that is installed onto a cluster alongside relevant RBAC (ServiceAccount, ClusterRoleBinding, ClusterRole, and PodSecurityPolicy). + +Once the relevant dependent resourcees are installed / upgraded / rolled back, this chart executes a post-install / post-upgrade / post-rollback Job that: +- Patches any existing versions of the CRDs contained within the `crd-manifest` on the cluster to set `spec.preserveUnknownFields=false`; this step is required since, based on [Kubernetes docs](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#field-pruning) and a [known workaround](https://github.com/kubernetes-sigs/controller-tools/issues/476#issuecomment-691519936), such CRDs cannot be upgraded normally from `apiextensions.k8s.io/v1beta1` to `apiextensions.k8s.io/v1`. +- Runs a `kubectl apply` on the CRDs that are contained within the crd-manifest ConfigMap to upgrade CRDs in the cluster + +On an uninstall, this chart executes a separate post-delete Job that: +- Patches any existing versions of the CRDs contained within `crd-manifest` on the cluster to set `metadata.finalizers=[]` +- Runs a `kubectl delete` on the CRDs that are contained within the crd-manifest ConfigMap to clean up the CRDs from the cluster + +Note: If the relevant CRDs already existed in the cluster at the time of install, this chart will absorb ownership of the lifecycle of those CRDs; therefore, on a `helm uninstall`, those CRDs will also be removed from the cluster alongside this chart. + +## Why can't we just place the CRDs in the templates/ directory of the main chart? + +In Helm today, you cannot declare a CRD and declare a resource of that CRD's kind in templates/ without encountering a failure on render. + +## [Helm 3] Why can't we just place the CRDs in the crds/ directory of the main chart? + +The Helm 3 `crds/` directory only supports the installation of CRDs, but does not support the upgrade and removal of CRDs, unlike what this chart facilitiates. \ No newline at end of file diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/files/crd-manifest.tgz b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/files/crd-manifest.tgz new file mode 100644 index 0000000000000000000000000000000000000000..13ae7faa1db969aa68734a0c54d132b278146582 GIT binary patch literal 308573 zcmb@Nb95zJ*RNySww-j`vC%=twr$(CZL7mh(lI*Dj%{>ob$EB5bKX0?_u{+nA9s&Y z*rRICnpL~j`px;wwTPo(K>qrCoOM~d@3pqDtWW>O^q|rl|5TlJmZ{@);=KCf!ejP% zROg2cwP$A*XPA_fh#VRN$XIb+&h1)$D89H11?b4^;MCb0>Gksd;jr!2H+Qz5I#I)~%~{kMzpJ?4D>_k;8v9>4eB=|A;fp?ur9 zqVN=9BY)e+#lx0M^IYX5c<>TFb>c_7e2@nYKUaUmI7ob-a11mLp{~i8&BS4L%-i^d z@3qnM+UWe2-}7|y`U zQSBizHIjm*qNL`NgdVBPa6L*5vr}Y%R;>?zuMjftz%$O)FZoF+#=aUhQ-xg?c4=gz z9Q)_qul)G8#Nwh(fSm`w*^(_0D7hsFczes-LE+ZXjF*O#r+n-zYu} zSH3*LxPIw+kIpQfgJNXnV0@0p#k^U|gVs|fy%#lAbIfs#ll;(~z)iPTZ;nNoO0N7J zP%u<8#&YTPf*vB)FXI$+H%=)p^$aWG#>>%|m^(J)oRjMOy1xD8kmBc|w%6CFx#-(V z!5*tOyuy@-kU6E3(iH+CHZheP7>ZvIy7RY7UQZ~MLxppcjAk5GA%|yEF;W#f>Y7nh zgAByd$f%5mE+UlDs7y*KEYjsEB~ol9t*p!rVS`FiNzsn6P5V|WCS65Pc;yAxYb||* z71!D2idC&Wgh{#5NvpJ?Hc|UIaD&pLktCr!D2#(r{2}mzR38+P1jY3In0Q=cA(-&} zl)Ds$O+oLS!;JGs^OBz!A~(3{3qlDWz|1ju=I4_3D0jttAyvh}^ZmqPX7Gn$i-uBi z=Mk(WNkkX;F)@OaKq0)MjnVdSPuD0G6sKqw+eYpQouZK}h zsfaESO}k@;TAp)&M@>7MG$V52Pk~E0^n4d>m84v`_@2fV^?a@B%QuQ+ zoLuZ9$|>mwH2B6xNX1W#5>`e7X$7N`OT?T_i%-FO75t|yqeyi!Nz0J)zjCRbZ8FG3 z=hsrKALc_~Gxs1(*ZCL3v~RIv&12r1+0B)s%LZb9F0r)fNz{y_!K9l?L<*Rv-G0H9 zba&30;AqY6cbvgb*?}EhoGuXB*TIx-}5JeRSm8m&1z;o(|+etBk!^_sZSgY?C@v zuWmf$NnOWp@?1&I45(9VZf$-gh*DuUGH8+)8#x#eqdYqoTp87ZPC;{&SxSs=Gnl0J zE7tUcw6cSVe#ux(fk`F)c?)So2XW55?`B?)8kj7odjQAQF#NMk|PPx~i?0inGUU1cG>e zj!#ifC~j87VzbLVOAn2zL#Jaxu*NwJpXL_+lCOqy5sj{+OCcYWjm}i#Csq{IxQLI_c_cXPNS&J<+!{?q0L^1%?BjP@ zwNHL+7eO)aY@}MD(&1v`WiYrHoFrO8&C8NP&4bBsMW!BL!p$~(v(#EpsCPY85d9pJ zqn@O&`r8ciXZ}qOdIOVIHpYyE8kCW@{B8gp_!36B7b3EX7H^q;TG>#n1a|7Ix~X;! zBE3D1RRV=qhln=|q53FA;w-V@5=iE?Sy}j}&1Mn-tBG$*T9}G?O>#uW26kgW3 z6Z%wQ^P*T@WJkTQu;>T0dQt9kTr~+@&#KUl=GASh$qr%Z50B~JGw#u`S@1XVZH<=l znNS#$YS3JSQI>j4m67E?S+ylQ@khaC&J-z@qsqqDz~yRzn`|T!u}rCppHyh$>wuII z^ptz1<`+!+4hTtzWS2_{$7hq5au(`$lTuI|vU%O9pVaF2J%Nj<-1hn5k0@0=RGJU8 zw0HRx`U*U{v4hK(5(c`bosTiY9>y9=!{7Nkel_fN0he_s9E?jg)GaDUOfF61IHJX! z1_TbdU48lJ9)m51onrXR63-`irHKf`8t3pUKwF>3GuV?&kXsCTT0uaN`_72p!*1&e zRndqRmRTMkm?!X-CGb-Idwp7&ZRa^$$M839dXAakFOT21g4IE5^` zDM|b2{r+Z=CMO?|+8O#K6s&@;=f97BNB0PNKi`kel~ccbOf^mCy}f4IZwF$D9vv$c zu^&3nUauE|ob6dGc&P8-3~09emQ8`>D}kOTW&%VrAL?k*{tl2`7b*!+rm!PFsB6KZ z*3nq9bd?hck42QV-9zDfTf4sP`8fBud+8B;`F$MSLl`gc@zP{Y?f;p`*V6xGWm@6m zdN(?G&-fBclywb($FprgN2i@@!zKgLZb}kF=I-dw<|o@tT!MD#nmEBe@frPjlZ}68 z{NB3Q1~WaTbYYTGYzHZ=W?)?v*^q2)*_QejJQe8aFvwi}f-Xb2=~1p6D;Gy2IhCPl z9(untOjN4oZt01-URfmz!7k`Dxcrm5M7?Lr;=; z-pVeR=}#(+MR_{oU(dsroz_7hOqrI;TsxD_eblaLL6by}pT$~;(pP-3KUi${w5 z`C?kI4Es#0S(@V}oZ%KK`@C^*ij%8yj7YH~Mg-#`n?@y++8|=2h11eM74VB-GjTNM zoAw>>Bu8+1IvDBD*qHV^EK1rF#WZMUEZ<0VPDDl8W^U;S6qh7a%E70yqbIKvHBwTvL#Wy)-39v>M{(EMS`!k$Zd1|> zKZ3lgFiv3M13sht)ZP429$=ZZiqIsJV3}G-OQ}gW99yz0n!9|$)q`4P@qF(v2st## zJ~y|NZ_palYG}|8aWnPN}Ko`-=pCd`|&SNv%i z!$VNZw&rWxcw*{M(g8cJhqA5gfE}5~iGE!@vrW>qR@iBr2SVL`j^nd^EW49)4(FD* zbI{y9pN=U?n1+ojRm<+9igk`BC4G*mtk~vfaYf0Kj}S9rqhv}uC9Ttdo>tq` z(jw&!=*FNFT@xoyD zqDr+?AYlw5wQ6)a^qrCD2ua2)M%`ElpOfeqe1<_^M|uVOAtxO`;@(gzWbez8XD!HM zy-sNDEECI?VoXD2ru+^m(?CM51)~H*cN`Qm6A$@(9f_4CC;&Ss)p8xhf~Km;#mSQqKcMx_6v3IH^bmec_}sj4iqWOnx(?rKcMuJ?6|8WErxa)*e@dzjELp%}YxBZLrx`sO`I_zqz4U#?=M zbkGxvd)lbj&c&YW`T;*4>u6GP zjIUFd?O3bH&ke)B_RHM+(+!&BNt&0_FGXbrM|aR-feMJoEfS<%$ylRF?fLSr^Sol* z?G~Sdp~Ue^@WqgiAihe?FsGi)g>>PRTgZZA+V+%Phta{!ke&qN@;s}W%`9@#ua4P? z&$QAlnpq&*eh=AvedECOGIp zh|1z^@Ib$e-qf>dMhsrAAO7O}JTi?ZN-}R5`HX-aTrlndOTc>p`?+?FKFv^MgcgBh zAA+=8I3Ph8gp!!$^>Qp)-yrTyod$86z0_=jx#+j<9hJ86hpm_W;>jyCTH#xKX}?fO zUET-^ldLc0mdZ5ed*>ySKJ?Aoi0ND`YO8dY;CI&g&1POujZD*b3_KhZ3f)XeukWpX zKHCYwb?dwldiM=4L-%u0@0U-UeG$RJl;_gy7vg6q$--8;i3R+)I5$$h5agkqCs zUtarpLW7AB=ldn^;m+Ckl$;*?j=MfZalXE)nPooSlOmVU)eMVLiPY)ZCb+&mTivoL ze^T&RngRbYWa(LXb*eo)sC$qkJ0R<#*#o(QKdBNXbaaSKqgjY}HxsZX_?o=^0Ii@% z{mRQHy!~)l3b+0qAJG@ICP)yl;$JdBwEn)K!tc>02umN^S?985QWLg9irYAO`%LfV zZeS{tg(GWqXLte;(1(g(;7|i%0-)k!z=eR4WL7nB;3I>Ahmzz5Y5I`L;DR*ZG9i<2iXRP@RC z3{&TGZ7?O)B)LMAK6sK;%sR7$W6!}ks@aEDy@{8FsZElMvuu^UK2p6LkwYK#Yhm!1 zI_rsl*iRekjQ(!#4q)Jqni)naeBd}8ieTzR^_apKi!R{x`<5!HgDHTla7wJ_e~&^u zR<}wvkSfZD^I7&)6!il!fb3my(4Vj%qQ}gJ)Gj~+>*eO=Zt`eyB|VR)#XY}(HVQf* zA50L^?^){U*j*S>fh*2MH{PYks$~~x=0~>_tR%M_52|okWjfrP4rQ6^78mAH_FT8+ z8P`(DK^jB+m&qrzxw=JrTaPx9oVgU3kZ9A`Z~G>~nv{){Xkxso9hUx{fY~1tQu1)P z4O-WTkUf;+m-i^@w$}NZNe6|MrXL8SRd<=c9OG_<(tqu% zDAk8}^X~L^*z$nZ5z0kS=K1a}lBN4{x}=^6al+H#=X=1iuSM-OUSjKTXC9_>xL^uL zrAv&U0{ruy8dJpFlNh9oetbzHm3Io&@SoF-+(=ZvLFG{tk84+Xamy}My0o+woo>l- zR+~ILvXf74U(xO^?bjLCY{y8^_?7Kk$Vhp;xY8 zOu;L*3xZxiR2&SNddGC4CvDq&*IlADt0VC3zP7F1b+rT>tT?+7v0!gjDtXwlj_JyIh9ox2;+{|p8_(OxWw({AA0pcBQYZ8rwgN^PE>a^HN z+d4MR+Q*X%Enco~-Dte&cF@skbuO1I?@!P!e16vpPafFrxeThLZMmC(kWYPOqS5t< z_wl0&-W6x6WF)+d!q~y?y2RPz4b>P<*+y5upyNg{pL5IgI0fId2>i*sIDx$*PdTtc z=wBfRwI=0J5D6vnOnyT_$M;vP;394LKVyXpX2n1>hLH%YAdDcH4ie)#<3K;3qR_*51u#~nT3OOFCtqnR zfr7-`MA|ww&!Psj3D}QoX`IbBl@K9|U5p^9^HMd6MQ=@;3KGx(ZDl2`tQnkKg_)4i zcKNXuX!;QJTAGxnv?E!nNVEaz-pvT~VpD*wC$u%c)%tkn(PGyjw{t}&6T9$SCkcov z*nFfgh$~q@5^{ySimJC;|-CAqwh})57sNA?IBQ*A5V_AyEx=x*&<74X&Bl= zvEXAx844lG#T`G3D}#zIV^`LW7Gnvt@q2~7q+ct`io zm&qz8!l<^!b>RIptm9mEFV7tPv=wl7K=Epwo_;Xf7iO+92+&iZ4|t%+zG_%a7Gkc_ zt17@=AUA3NMAQAF8eU&rmG1%GUvWhZPTl0E99UsGV$ypH_wpL%I=XgG1Vl$@Vg47{ zg)`Fw%C4&4|1P^(*lhjE(=8w0RB$3FbfD86G?U0&IrL&bE;`4T0(}IXR>b(sh%a_)er2bg+)vibj`s>s>t-`+Jd!5A942RG2Gu-nf1H% zwq&h-57aWB%?iii#A^?q*MVh&}zXOB-X`bXI^<%b;Z!2sYYB8>!vLl&9*}t5O)3!FUaVKc?Bp>t{y?X z=UcLv-`|K)gCX1e|53#6Z=IE1S9Tu-9p7oXIqkL+Tp>4XPxr`JxO=y8sPIe_VP1Jn zP;dykR08|1BXAgS2zEdw-pNfL3>;j5PZu%HwhE+}UlmM3yT(*Zd>gURS2a*nG>d(0 z)+K})KKAFerDt|NF zO)=E(Xrg7Y+ZTW)f*lfXjb1=^)I%5K(4UC{WB$f|YNMgNqg^;wafGN1QR40O$VWdE zgOOsRZ1$l``(CrDTi>)5OkSqeElQ}wu*Ls*(7GO`l1EoleYKJ&cM;Qq^|R9K*OB7I zfH(oyY55gR&-?j zX!?1->|TVro#c0l-v4c=czxGP!p+OEZpc=pof=u5!AO=L^61x$X`T?M7ATDnKL{M4 z{tM{r#Jo4T;T}tNG*wx(0%ImK(`32H!~9w*HoV8)0Q)=X^CCdvDQjSE-PAwj(l?=vVvE8*jO) z&cV=`IqgP1cGJszQq9!NwnML&cz!@f{o%?PyfTqSLpTMLs5 z{it^&Tb8o&2^RmyD-Naegd-U&vX%bqehBaLLw0? z+A6+%S#k>4sbFG+uU!%yBWi$PF$(UvUu;K?i4)RVrldyp`K7nUR@t?(F3LVO)A)=( zBDzIRJS~3SUtg{q%{JU3&s{1tL{YP{eJF6ZPg*#h-gGT!U{0E~WE)v<*|IqLS2rS^ zJU^S$vct$mE*#%{wpj{q#6Pd1Q)Y-%J2V=eKk~O?;YL(CrHqy24{nq->7>G_Ze?|e zsu1JegD3}L4P{cu=Q*WFWgXL5a}Le~IF*FZMj?(|%Uk+zGo{ho;!Vj~J4nSe2)>)Un+^@&XY>rkM0|;VCBNcwml|EMpp-m8MD6*#f=aQ zrV~mLtgEIVeE~{{xw^Z&FkKb#Pa_2V(;69N3eqv}8o_`J0>{H~1iuGO5233@4Wz5a zDnL)|4Y=h4E}WoVny%n?jX`>9fblB$f?3B_gtC4G1T*lZMyss534 ztBF6NV?6=9Ag9m>tcW~{Q&56(G}@7A3)o-o{ZJ5PuGRxy;xBvYso1^e$ael$5VrjT zHxPkqJ?V4~FM`0fZTH~-V)l{9=I!v}u6)0}@eUZL3z7LFdLX#XpvbaNfqgTwm^cJ~ z($GFsq~0?@B{O6wNnoSh*#67PObb{x&d5rQKOoAIWoa|42KKIAbaebXs==}&%Avh|CX#CT|QYtx+w20QR z!!EuqEv|&0ls#aW8%p4dv=bn0Wy+b4q?bLT3wfyLj=n9x3U29kzbmAiuDWqV!q5-V zw5xjQ0YE-!tzY=)1X0fnbnHs&is;3gh@@0uZ=Md$u2jr+uNj_yA9I^@)SKIV5I~S@ zsCpBrUiSBz%yKkZ4CzBVrNQ@Mj zD=N~V$CXn;H6cbp^;jhdV7YoZ#Y55N#&=O#Ls`#Ak*!=V#3H3=vmrjRXYC+MCmkE+b8b(~+HB9Yma9yQ`EWN-O)8$vSo9L=7>x#}JjRcR(N;xdjM z_IT=|6bruLS~G;>j~Wht+oC}_TwBBIi?6gcoJ{e@Ei~@wA$7>=mwLqWZrtrw$otK$ zEXv0L0g&l+gC{|r9YON~)r*R_@w5gT3C0;rhC)#w$y)>+L`O3bCe!ZMUr~RUHLkzW zWnbTBIHW2}Yr04)HTwCvzzlox3q7lPxmUGb*~5l!Gn;K9$MR^ipPPlq<;uvspCOm; z7v-Xqh!MGcFKgo$udd=nZ>*PfkO9BGUQwaV^pu|nJIrTDrZpLbl+_;5dM=VpU#g9p zui7r=WDuy6l$`^Aqo|5Ygj7Tiljp2soiULgGaM;-c+}6K$x3@T2EAY39dE@+_I$k9 z=QHN}>VEhvJ@MGS^4Y)UaQR_PZbh5x`?p5BW{&fPedZzTZXv20XU9BZUb5Gt4^%bF z@Zw5a8o;8+PaPEapwkw2!FAr+MotfGW7 zGvz!qOw9_)G?-NtmCsOFH7aPa{%K?!Lmib~M;t@I3*!Hyw8A5m;;?s2I-qw|hY>L6g5!F?)9iyvSiRy<$SHXmA> zVmB|DOJB)JR!&>_2nbYcgi_CBf8xhU<|!U!kwwrEls0YUK%Qm33;{nrYu(_S+cA*{ zrC6u$cGNU;6N4_(&Mq@cXiC+Cy~V-b`i><;ReUS)_%S2sC;G`LxT0bK^C%IV->E=S z9=<>lsmfEFg1QZMi~Y39CS}XGthaCh8>01s zTyBtah~{Kh;F`ljnj+sVr$t|UCF!fTC(%tS+@*@(LxsUv!BYN+gHxwmaS#3dE>Xv; zE)bB}U6&KD>-!7xvvyiugrm#3rbMN-l&u)CtPwg>PQX=mntHfV^o{ZrQ;mL)gH6^s}UT|~q@{}Mqj(Csk8R?mhHJInQ4cj%VqZnJpP(nNFiA6(Rf{73FC_z)T5?iMM1D+3>w(=0nHQ(QW-? zD}}rXPgp8BhN^8fx+YzGHpjyqlaXRa1KEC`GhkD`SW{g2#&R~}C%`w%rb_qhP)K{9 zqC{n1CBphffV-WA*J;A;F-lK@B)&VO8b{BlMhXqm3d=87-^|X=D7;$4=7~i|3=ggCaO}AIQ0z7?QyMlDFr=;|o zIgYytoB5d0?YvV3g}+UyJ}}hO73wg?)LjsKV+SG zlep-h!4O>x=LY-Ap*bSbCF!06w<~Gf*5rsU{cOYTY?#WQn>vNd6wdj5aG?4`T@7*e z+O~oVE4O07_fnrzhs)#rBLeI1AVOkC9W7ECMHy2GU&hkHVW6oyv*Oy4NVQ51!9X~i z1{6($9CgT)8mu^x+rN}*;9#1DrBjGQde(xJ0D3cx4^d^wrRg4a^wi1}yLX2!5BLP&S z_>=KKsyVU8(T4Ph8vOQPUucrpAGw4_cd;X8u!A_cGeJd=$_Su5L8dSD}$=;fib4Y!t+1zXoc36+=6Jm*49~-5Awhe$q}0# zSa_Uwzx?S#lxYTv6C5}UkMM{k3duf(&?q#*M=}{BF5^Dr;j+Kv7>K;G)#4jj)|R+^ zBgey5*<+dKXL`PSofl%+&_JnvR|nX1N?Ij#&h@9u?oJ=Vkw@}$If@#Dy4e{bAhjB{ zbDT-}KTxY9`P|?q5j^1~)01+{x2@1r-sOz1|5S^t6`{2CSGm?RzA!2en6Y)RNYC30 z4>IQ{b3e@pG5V;~1DFQLDa|j_H;dpc2Bz&z(>(QwZ-lCX%yi7i_E#66G?(h@Ihj9k z+=jRl^7m@zq@~BcD?ZSr>)`1*aZ!h!Bcs&`X6PYLwkms{_CVNu7=W~W!1bax!y=EP z1HbB3^`3tKsnqwkfxlDrzH{K?3wD1=ZWyCJm1&b3w#pz(DojFG{$egKvPAm-t5$@d zZJ$Y+H7>r4{LR`^Hcgva+SQzMC7N+ZwlvMhYX+1l*o=K!r^jn< z7qLarme=pYFC!gZ(jG=i80i}dp-sKemiiR zL!WwUM$~rbb?c6=pD245Y3X%$!C88EJ`bdNxx_Ov!yDr3Jt zob;)TZhfAVU-dPh<;hi(o`^-G(WoGYp8V-Yp!5iChU-JHMGK!X zg$keX3KV?>fxklr?#l&;+;4z{o)jwbV^m#&eHVhmmfDe@q2jCgEbb84oKE*$=o$Dv z>5w4)9T0}YmjEkc3MLd&&7qcYBkiuT$LBbUo|B5)jlE~5UJQF zj4DSznCXJ21ol^9P4&trPBDEebp^M} z%G#P}!&d`3XZoRGtV<-rl=GX-5)H+vC>_E>$U~LtajtSfoi~@~8{ml2>ills`=$J; zX*&^iH&n%AeBs{+%tF5rX!21(6a|b?WTsL;2}VtsC$!^!Rx##S$y?7@VqvpPCUHbl zL1GxhF8k+9>_%VepQL9s4ivC1n0?Y&CxSX@KkQQd6fv*zWUIO^b(nz_xJzQT&p=+z z=XG>g3*A*FQ8H)^seaMrTHCAVy%ucbaDQ@eP<Qc{lT5qSw2*%4S8%u40eodz-i1w zFF916d5iN0L*Ccg7XyM-wU4^i>>+ASu3Q#9u4wlY*{s(P&qAd2p)u%|Y{&+pybS*P zhp``JhtB|)m;N8zeeSzr7`r7G53V;0mmhoMzk2+yUzq(rUfrqf)5y=|Z(H04s@W;*8B&uw0;c6ToB)!7@c%3aiHn;^P(I8R^HFXU!ZtX6FR?g}$zA~TbFVJSF}384 zt|>n9ML7vO0@N))%z6yC%$-*QC;XE{>aY9lmizEdncE^__`4{p$qJ&WSizA+qK+() z^7feU?3oa4a^bWQqS^#T#srv2&WW3dCSHLb{NJFp$tL0=fa(1U>69A`!q)IA;tr>1 zM%Q*5X3%+vt!)TZPsGm3iKlA_PhTbMF_7vx*ZGoFg-_Qz-eg}>aowFW3JfA*`tzq> zY>}0%9T0yMWb68I2s~WP_Xa0SeB%>bQ~RoDUwqrNjhUnzjqober6SX;vAb`pCHJkg(_ zUeE^&Ezx^2S7FVZq~fVBR$4uMubigtLyyd0#CLoT>8D)v?yTyMHi$2vUi}vZGe3i< zk{s2dYh=B9qxhSjD;%Nko^dk9@Ft(~CIwC+S2f#^?G-wg;V4>v)rPm=>c`sl zhcs!|USu&$EqLYW<#4?icqoRc5FA%h@Aj>$+nRK0kpY^GAOX!q&;^!5^>vnzOqEAm8P~W*ZgCSNz8w-d zs0W?hLU7UtH!)Fc1F}8m?RkU-?37;*cnu?azQLFHK3m%_c3;5|#S02YpoBydfgH9_>fbnQAq+bP=zry~h~jD> zh~id2OK-p}A8_FW#fx$QXU7Qq%Q*aYk)Y09cxOIP!Y=&!#eq@QcF~{mg~QCh2IMe3 z1F0#1`B_vn@!vS?AGa)MfZhN(tZ1U%-_F7p6h{2#99EIyKjyH1yoLCu9P$^3Wygs0 z0km#?f50lXhF5uuhjpZcI`k06wZ|L~SJ2nX)6ehW^F9B3B$%Vu&F}y6OVs84A=`c0 zO^KJw95VF_zssbC&FIC#xD;C&t|LWiqX9MiS7On5G7D_*otmWPj3EwB?uGS%4Io8b z(ZC!AUY<{Z2@5xFeT z9w*F9cwBR_`Yqh`J)DVAum5gTuwFdRbYW$sw zyIvcLgqLd1ODao9M~PKl!>`%H&!jymo?6$TwSZRGB}$2xW305&w6@jJ>clzBm6qmP zN(eUf5ph;Uy+T~vaL=0!Hi9WL$MlT8W@#$zG(uz*!zc1b>MX_$w5H2>dpos~SnY*r zd!I%j%Wg$?eh}t)-SCc6YY-+d2hI(33iE;3B$lFBi~p$|1CI?+V;H5Tjj)BB3Lyn}}>L zIf_GFh(0JmK9U0K(V;I1|CE&-h#P?f&W{-Fq^=G0Z@v!+irq_1XuJC_21^{}V*5^<-B;Up@eDic}@24-^V| zqyW)FgAE2D4wyb{dx#-mA;kTYeX(%=fsatoe_8(r{s9RfC{DqJ*M6|43#FBIC!Tat zI&Rv4GMbKF&cQEpRI!|V9)Xn_nkHj>PMf9r~~2xmG*q< zn6r+1frf0n9p_=Ywn{WR{SAfOE6lIHsr{i$D|{}H?h=@9;`;l`9iHLYl5 zyO0#qI{xCcFyMb^Z9wBc`Sk@PaIi;;kfs&#ieehxm=ZF0O6dbwN@-Z*d^U(_r8V%x zQ{Z-|14joNp9;?oC8W204+>NV`?M6UE}9=GJxfFXltgzrG=S3c)`62!YWHu_(>PBL zG^3cd`of-4dI3ozi)dQe2{NYS{U1w@Y2|+`J%3sMH|fzl0hUBx^?AP3{<31x@D8RpxekF;7T1!3Leu9KEr;y6^~7%6uKEO{?iZZ0$**?5tTY;% z@YzttHizwHM|GZm{t!l4n%s<62Qs#L2(_Yds48ct$}A)5Se-E#v*281G}lE*>QKXO zPHG4GW)?0c_wlQTW-|M_L?0i|Am7CrzOdYsofi2vE^SGDw)FPO0Odybcdof(bQkPj zS;ms1ipcWhhQ_89(3UQ#3<-twAh@gGB%=PNY%0#mm!Wa712(1kyJ4H-1c9BThCgDg z+)IxXnMqeQ`Fyb1g+=p$-M;_;@Dp?}kK_U3TRCaKGH`G!0052mYy-l6ckGRa_dNe! zqa?g%AyMLgi;@85kN`kAl*Q2;GSCn>0AP+@P^iEKJk8<=9MQJ}1kP5d5b?Wr7cBDW z1qa3|kmtkM{wdvua{jAy50Lm1A_1V-?=1iAJq;6}|J%J6cAzhi|JcKO_V|zAQ$*kY zymbG^+kY$Fi{_6&+kVcCSZyrF{>3yjqM@nro$y2kDkP0{OFB^JbC9(?PnVESe1&Md z33i5T{2(GhtI5)G@+n%(@n_*DB*l%+_I?y?d`sz#>+rt%2mkmc88@3=M8a~-&i)@J z^2oK$p*EkY3C2w}`S~RTQEGN!c~Gk1@H$Ki?yHE&=zvK$TP=7gZDlfHg%R`RU~TK5s7?)o85wmoUp(Wni53|Fen zTf2GZQ6?L$ypw!r?uY6vnW*mL4#Ag{yNh2tu~jNQS}ObqHM;(@4v=EeD$oK^nzi=Z z{bz(+B;hULW^H{56;vXucOqoM)S1t!eM)4*N9%T^IU75ril0hcB;@Csxr~ZAaqHEf zeh$a2ob`k(8;oKSbLBXyrTU-j1m(smv+FvG-I?$a2?{G*+YFB)WPjab5T*Q7YU$;GHje?xZ~3R&u}X5$0o)YPi;tflLu8X!;Y;bFz)%|}s>K{o55u31B+X^9;UJMW9Y*gl2a}qQaRp;$> z%i--N&C>Wj^itWn$u-w0yZf@ZW@1{}gJto*UL%*y4LoT~d%MGr%eFWjS<1{4W3+j5 zMkLFB?TbPkmE}2VClgOaM)U|ldY>i^Tc;+!3*YH^uZMp5Kr%$0ee#`ZYGq$~S+gxH ztc3lMi7Hkk;NHhh#NE`;K6mQ;>Js%yIFED^ioK1n_WSvEnMajNjk&Rlntc&^ts{rK zl`$6)aKfGyx?g8i(H5-#$8BP^56I;^C^L zK9hW~Ns{)N6x74=(i(O5{5om4l>M4ctCg!P#4jGNA&f9tD$l+_?MnNf+{_7@-BnOb z>82tuTyIUvHIOo+ymY=7D3k5ubp7BHCCf}EC{98BNK8HG?N*XbOnh)go%wywojLE`HiZ}_YY+09CFrnDQV6yN zu5T92eR;@D$r!d(N3310GEW5;ZMA@@duTzWU72Dv;+7hE(S|AP;;o^voJ9OmT@$Qk z<7VrSSEwTyq6l5=kaZC5$K%O{wrM_&_SOTu23OEanFqkBm9z1^to3bv(^g}gpSwlQ zRweY#0bX?{uQdYgo6F2ESF7A=w}hb|X0@tZ66ca%B?_=Qd;{U=@Yj{(ex0nZa}Vwg z6U>s1x4Hv{-`7J!O<8z~Nd|_8eJ8N`uy4;1z`D9g&mIqQ%qo3-;MY1ZM*u%lm`5t; z?BLzhW4^GTiFwNuxqH_$oV*T{E#n_s;pg9VmrBdw55kUe=b^r2Z-MY}17VD;W6G7v zhj|FToerzsE??&dS6qC2y#M1t`r(Iu<_B@llgP)%Z*|~I`lnlFzuVX7dcVhm>-1?s z_318dKR=&We3u{5pw9(pXt!Gg#OU(LPTM@B*DkzQkH+BlAH=XeTGlcITd}dNDzN?N zYEnv4wcNE?%<(z5g}3g}d)JqGG4b&cJ1x4?F=%;22Q>vrw49&iaj3sk)W0^j-wNQy zO61q{zf;4cr~F7SLFv%FKleYH2pPInu(M3eD@adGVCZR{iET=?%SMk(oh`w^Wr8zn zCK?Y|5c=1wcg)M&|83HH%Ut_LYKPzAmjUWU8u9Z$SEP6JA$3gt)mZnS4&uhIFtS`} zCc_@*Q#gfj$BX`~O-(ayO2>p-cCW`AEeE~hHM`u}#hWbmJbO3iQAw!%?Z3epn=|Zt z?jmd54DYDV@s+%)a~Wfe=jmufQde)1OjY&f^_OA7R+dtE7iSc2^Mf$4H8jtSIhu@L zn%}B5(=N4p)==IFw-~B?M7}?p$b=-t$b25NP*Ts=|2SLCpLAzl>tQEWGj`iY_RB=i zVG8f2|5h{j-Az>2EjeKBp!bpMnzDukp5p!Ce(uA*hA~g@6_gP8eq+K;N_kzKT|Vrz zD6~UAQ}ez$m)mLZ>sNme-Ld|}9)_jPqNK5wkjhE7EotK9HH~m(oJWq`C&V^ZbqgXA z#s=^$6zvuLu1-?c^)ft2Uv(+VSi(94H8%RJ8<$$b-NpU!ep|g* zzZ6_PnNjz9T zmBsd4e8iN*$tSv82w9Chb;#6la>kgWlBAgE;uHocH;N;h5Kp%9B#Oq}eVBBY5^3f; zP~2a~<#uh6!AP}!0|hfT1_qERiY#f}o(sHlCy{Q$+du|-x*(w*{Y~J7ibAo6MoqSW zDh5m0$ku!VM{~zzC>QT{m9>;O$7m%6Q=-&glj)6gUx?&z9Ac*6@6dixct=96yntD> z%}dGIETge=B;AK|O?)cHzpG7`CUVK;tM%`fuK0WI{F`6FOY`Sp{^W%n3GK->3ghHr zg6hY%Tv3h*RHoU)oq%)CGvrNncIa8$I&0_VhdCH?+)nfeEUPJ33S8KtE^YcBgl2Be zSR8FMusc+{jfUKBXOI`j?9N&hL zN})xZ`?$_A{__Bh3d;TY^gWo6{2zji#)wi}Y&30Tf~CxTtwPV@dZO6R$fx zN}scMq^IdPw$$&#r_CX&{ohXbrRjKu;AoKv^?q8g5!GAt1yRy4No>jEZsvXooQ<;$>FGz#Bx zrLnk-e8hZALAMZK*>M;$jeaLIk5`i`U{R)YVAZp=JDnHXhNdug#N_o1g_41pUfLyo zpFl1x%G}{s#z9S1rsua~exvUQ7gjsE3Hm#pMpW04bAI#T6`5%K+QX89DKqt^c{E+E&apzCcS; zrb2cZ`IwBP2bl~JQ`-g-AL8fLEa~Q|HRh~SYqPkZ#3BhB|N6(dbrpM_&GA(Pc6eWs z38zYB_VWM1);kB++B|`uv2o)Y+qP}nwr%sqwr$(ajcuOTc5=hb`~B7K)>duRsb{(e zeddojQ)ha*KM!LGDv5T*+05XV9%LJCgwS{-RP@O(;ZF+B+fLPs3Wex?G}2wF8q0Jf=*74^;a1jx7?GY4+jd$(Q z+9g;P@jn6g*1GMCuojq00X`oT9_Umc5EKM=4CGWGA2bkw~yD^AneU9 z&b^aNLl_NVbzwP7j%nNjsr{l6D~$sYCO3 z=KCBrovl()klOUAR@6Y+ditnw9-&ZTflPK%FsrK zg0=_mz+k;sP5OnjifD>y*l`^r_IMz-m6sfBsRLLCsu3>CX2{yyTXZe4W!m9l6K?gZ z+gllzQ9k2yYeWJBA6~_dyIoDK;9t#njwyFE<1;Jzv>ku6K&Fw!KYj(RiB>9V^A_BF zg$jkKRW{U+zdPko1~kNfYvqNC1zx+>X&j2M_*#7CHb@lH@^6&7(>Uo7V*5rpkJv_+3-2=2vOU9XjBxo*1H3u&S|ec`R) zkxHOwyXP;k^--{R_r<-562i|1m>28A*scofAeb(NZW6SeKXuAix%&LZ4~I9G@R5cQfgN`t zCCc1j!#1okz#ht6uCj?5iEh}ebm=Og;}MGrIVvFU!YftCTD7}KpiDBu?kShI%0MZV zvD|z(c3oWS71YSbu2IG~tq}XVBx|gxaeQcyV#Zn}E=26^;%cB6V0;r|Y>!j_&Ugjy(%;s#NBD5n z3B64RN9xB|@TUNpz^5y&cyJ>#9(n2L$7s=F9bIx}Nt@^nj zMkT6cg(C(L&SgXs(pT2zNS1ElDax3-TJv;o7Mc#sYyX})GbTANTKx}L(E@MY*UL9g zm9QemBvyzZ15A+OujqSFup$=3uZdFDtqI=RtD{f=t{HaSJLL#BL##E5H|Tpk z;KO1|cwxouAKMh4rgTKBI+NO`|AjUVrPq0O#gOkOZaTV=E!dypBCJU&*>)4lH8N{e zXlv5%l{Sgb@F7@JcOSZMcHMnQLdlt&71@qskw3Gp96ww3XmyZfZ zHmHo|1A|6UQf}_tbtY}8J>O$6>)_;V!mQk&3ez(}U6}x?{$omO_5erKTe{|I+&4qH zP>WK{%6iF?!sw$1p?kNwb(q}yV({XiFBNR?sJp4~mUwthL1iPD_JsRec3kU|T zvY-wT33qqn^C`D(8?jSxuSzi6*5(H5(N74?+0sY0>n?D|%+qK(i3a37`QawRlA+4K zhUb%|R!ret(zcv%Di{z*tCb=FnYa4z884%P)xa=8xy4$d@k8p+?5M$k=EAyXiZ89t zj^@_h9ZaMOF?k}L*nNKI-!z0W&Ua4OIX}_fCAV|;2e_52(;(Dp~^@E z!lMoTVT4Xy519M&ueEh;1#*sjV-VJX+{=CgIR_mB?;~0xb^mx6Ime8%dG>G@Dvtnq zKl-y%Z%NTODdAQ0SPavpm;wUM-#*&qj9q%ggq|lkoJV?7zMLn){|@!4dH)>;cO3c! zw^Oh3d>qGN^vN;p{Hw`ppSt6s-imTp*E|S}R0!dKSBJb#AjR`-GN*6zdiE?jX=s{@ z(dOwF#RtZ?>Wn!UUU$_Vbw?SEa!Fc!^Q^$IB(KBWFVTy}2d<0hjcLlu;!tdJut^?? zVtIs^ky3-4!*qKu^=%@vxIiGour-79bVS(h?3j(gg2Lnoh!2D$FxiEK!W0+-5`y@} zbP5E8Sq&TlxALR)WyTA>>c)+Eo(@1x?U1Kntm1jHN%N9I1OgbZi;4+)kw+dG=RIkQ~Pxw0Tqk(P;=Xn1W9= zX#VSORQ%w5V+9`?xcpZQM1ea-3O@8V1XO(34=_~x2Ow1ZrrQfTK8PF#JaudE(~yVx zN7aBID~g8TbI>P@jn+R^jE**^A)r3qxcMjuE`xz`9KWJ}wiV$2wiPRnkpC1^u%d3D z0SYKFR;ZBw6ro*0$fEzd_Z&1N*PZ}5&d;1~r^MHw%UoB>+Hhy)2(W{g3UOzE#4w`n zT87R@eSu!Y;_||CIY2G60ZxTp`)~xhu9Fex5zHJ-wc2tv^4Y+}R~L+myp9m{D*&f_ za)igffw>%b0l6FsyS45L3cCm}TY`aIdI2Er13=){gFrA_o_Jv(?$|#{e!niifP8k? z<{WS*Cr_sx!W;_Z&o=-){uTU@+A02LY7XV+n&7|spS2~J;;b9;wwh>vZc(KC zWu8JZf>1h~E!V7P{e~MeKdr46kBtI3PfmG-5`GR=*K8!-RxcZUh=T!7smaVa^yH;r z7i$$gW8)DN`F!Y>XRD;bt1BS3O^RF4=1Ut0fhoEa=za0x)z-SQnIdIyoxQZBL~A+9 zCQ%`=7IZuFy6_JbjfI;9%btmRoYu%a<^rMtl5%Ps>q70yG2_`LQ$zeb-wox2#s2#4z^~ z;WBahHW}CE+mHEhaK!Vh@3uv)2c?%-_*i+4^vZTuNXb_jLkRB!U#Nd zlQUHd2#Y7Ke}~(*yP451^t9b`o_yTb3XYvqp9rp;3cx!@y?*v^&1zwq!-Z<`OFQeP zznraXXte^o*2+oeD(xgqquR0tg?Uw6@K0oXw!*F1JpK>oS5IdbNq#Ib1V-%4GFtLYmeQjOGD~6t2uLYh z!P@zWNg>6>65%sQSNGtv(4CUimFW&d_BupIPgH`7&B&{!5f*z}^Vud~utXz}36%gE zW0Y+0mP+~bKe;%o7!>2>TuH#F zA&Lb+>P|BppYF5Q7o?}n|L`(VvXOgRt8CuB2;@v4Tcj{%snyL-d=piJ=xp3y_zeY7 zo|XgP^sPKW^$ariB*o(6fBuZaya|a-=4;@@>36?g)f*tr2FSE?5hd=&tE8_)V;xLT zd3PSeFojtV^R7o>)RZKiI~h?A%77ImIzQcz+%`(m73T) zv&PV|z}ECOr5Ff`HUBF9rLq8yS@aX1c7Q?~;OC=MCJL61`XAOQ!ZpxB#GH=m>mZ3A z=|5J(cvAmA#I=(^SXReD7NPw*u==jFRq{^WAx)G^T3Mm0+_&7Ug+*)J=C4-!_iD<@ zuEg{IL7f5~mdvi%uZOeLw01HB5Z5HtlDlvq!dj&h-#N*{?+t+z5ej*L1dM9Y z4rtc4x{}UuW2djB?ENBvdZqwtMFm2NHpND|F@D3(&AACo32V3u&p@ZAhkFzfTbwkcIidf!PBWlhRSX$tEjVG1J2CFr_)~nJA!Ir}g zxt{P4>PrUxQ%GVDrbJsr3`f~)JD>JD80L?}@msuTv>WM2TVstvv+tj0jb)X?la5qjpYHC*30uP6Xa&<= z8qqYKwZ*bmQ~!!)|Nd_a07hj=Y$X+Lr;$`-CuHv}`j6*ydlX)R1WhemYHr(o+>+H9 zDlHtF&Tl!Z(I0Oe7lD_%Q6v%)p1bQ5u2_Yo_gi7{9XHNZ1Ai+WgI?eMGU3OVVN2Exd%YUweaWW%=qs|jetogc~^nda>WAC7~I}Ep% zA(;b^zeO8mUg_j*;+;Oq;PA6l zR>O*HHyK0MXO!}o7-Y+8AQgM>(T1h+AP;~+XxuX#gHRX{1b+c8-|aUGoJRO5fs=Ve z(6awuf|GY0G1@&dmCHQdkb-t($1!`Lmg|%jAn`~|k{qy3#0I^s!hb0+3@V4qGM9tK9UPY~E0PE$ap^1J z4}Vh%+Do-~Uj>hdC&kOYf_`CZsSKM(T%`~UgMw5tl+)4&1eX0##WzceW%M`;MJSZz z>OaD6?NYPnkDKdn#8M5nD~5>jm%HlC!LL0+s-x`kh|VGD2DU@s+}IM5ZVth353i4h z#j+o=quEO0B0h#mfvb9q>k4{B^Vb`qOL^uV4NfRA~BZm8OTdm@eeWT?ntlkd@a)?+-lmZ^TYIbB{%U03H6e( z_w$_Avon57t(az7>RIZ~Tc40L?Xu6WBoG};D+Px{9lzoU#E=55%1uxh{?PtmxeFB{$lX5(F;FB-tnDJq<^bs1mNvyCPX)tR-p zy$|7{D(m_ade#D^JJhy<|0`#bkbX@{6{fDmw#>ME(G>vl5^I)mJW`y_#L5$GXLZQl zD%`ZMldamQ4z1P+`NOp?j7lZYGxcTP?0H{lQYRgk2CQ=#sOqZ;HR{C76qBFFDqnxT zyFIZ@+;I1HHUu|C11uXzs~fAGmm6*|@@-RdegYSod-AzyY~8fG<+HByvm_N?5{wf6 z2$8A`nWfwz&(#HyNfzC5zQF2wCkXZ%oE!(g__Boj`ED4l6BEah)gqMJsu%uCmW zZM+$`Tz;aFpEa;YG*0d093HZ-nZqHw>N-9*jcOjWRgSMoiBwfw8*yGwz2f8LxNQTN zQ)INYu$HSQIN{87WvF;2`hbQFD)(A=#J5UkrKDtTRaqoKaP?OZE&ba>N9H9aCAZ{N zLavpLgJY2m2^o$qgq;1nurp4R3_=6VFN+~H1?6PL1Jygwed#xCs-N&;DuzOpBP^s& z>oV&G7l|Gl&*-&z8lvLVnqT4ifLbk(w$ttN^?vdl`1Sm8b84w6%aqCEjDN<)zm)V` zgLaY@ZE7SFksXVfhdXVWi+g8Th__+({52JlfAk-K*u`;UWvgR!E@X^b2!>_Ro|q;D zbGZb;#atag>^#9=xH=*71Xq04-teVfX$qcN{vS;zkyhOw>ReCNKB%v8f)N9-q?hAj zb)jUl@pVj_RVOnALyhXz@Jps7J3>*qiAu zj=9wY76<@Bg9q|o5aseSn%prwbCOjUdj<8Xo+M2Qm5*rgJW@;%Y?NHsO^|5{u0o{U zln-UjWuF>jB3<;Tg$>lvLEx-4@-2({@hKa>cVK!aV6Q%OXZ=6%i}C666Rv z?g8mc6Sgf5we_5n%aeUpVNGd>BUjP+YT;~0g`4xBR414C47J|oPP+uVyv+@I-`Q)p zW!1>*!qwx7gdNtq)^w&8q!xB$PqKEK_-yg1*_#TK)>Jf34d6^mfeN%ky3sg!G3b~B zTZ>Ouxkw+T55tk$ZP$L%-!tzXi@d z%L2LhQsWj|H7MCYlsv-{Q}ArtdtF{;q#8c(Tx_yRe}WtenQGjlxYiP zE?p{~r?;ACb6FRR7G>urfhoXIcxJ~Md`_*IeqNADO}SD^eWo+bc=aOHEOZ_c=~Pk& z61?Pa2c6$>IU|nD9*^eEY5Qp+rrJj|$M|nK=swIMci2JB<;5;7tQIh<)DERTnxx>e zlKAAZ8WsO;au)KIZ=GeVdAJG>joz);$fT9?;Gx~ZQ>)L}mL(};_-Z>&L#FP68Y7Ev z=;uRFe4^@w+E_3&vX3gBJ%zX~$Mh1GHwMZia`vJLbf~E3Z0eVFti4jJh1}h&?<+R} zMPmhjVt3uhvR-|P(q{(?zXwS*PubL^Cx_+t7|CE>;_@L~~c}s1_I9=ayX0Gb+k8`0NsWZtM0)Sr-|E-h@<} zS2zq6_cz-_Gw)(!F@vzb=;~t7z!IAKvF@DNj^W&gT4Xz^SgEk`n9ob*b1&NNIY6(c zr}yJxxa*;9RnKlNYGa60s9o?|?)(VZ??-KplMZnR{_>rlB@Jp98EOW{H*JeIX+*X^jrM^nuKpYy!YFTllEb^p4^< z8d1N%lz6x!T}r(ph;*^Ed7t4+XvlCeA$4IkUlsaap3{33hX3yQIQzI<-n@9QpV8O7 zCh1bwCyeXZ#Tsh+IYl}?C8CscMcHVc)VkWsoKsIPZPaa~E1(BN4Z0G+kNL$R5qy3X zlU=GsuQk|H%J5kDMw*97+!?^B@nZ)K2Qw)_x9krGFG1%ug}z*kqsXsTqzAvqZuR`3 zPLe-Yvgvet2xLh2&FUPeuX!_zl+^f??Mif85ghld`pIjNW{RO3OHUAK&3ve1!tW{P zGf^ELTR2qso`c2r{2VevHr}1PRz`RsrVK21m*>If(qJS?3NW5k8DBs->495cXU>*=6QOe@_*NzeVoR$5H>uRcW z9^$M6LFeXL#k`nyYU_(^r=FJTV0x(mLBP(HZyh%IA_Axh73?f%DTiW+1m1bsfBaXE9HAq=rH+{xj zX&tZgcU+IZVeA3h^T=LZkdhMOczRc&4_!*RQe2dHqOBTD1nI<#^TcyR1M`|L&J0Kr zaOl$U@;l)Bj~!6(dJZtd;X;~;W2GF;wG3iyOz%gh_lccH?@b)O6>#}Q*eu^P*WAh( zqVsDUf3~mV^Z4gMQ&ZF8;?4K{aTTI^o&TYEUN6SgF~s-N*`xRK+o%kYxw(Ecc|ELl z9J5n_d1J=j67f0bMf(95;H10T-CcMT=hOBXO51lr^^oiFq9fwhsS`zv?jHqeVkYr! zcHOAG_qQtz9>VR|kzjw$gMkK*L4z)S`KdPaxuTB@vcUlb~0@(a?k@YFzn>( zva$9E+-$sWzaigjd&r3>mo5PxIkc_*2l!*+PBP}~3x3mQmz}%``zD0*!etHKkB`o- zL#aGLtVG@fn)B-@gS(vN13JvWY9eCdN&D>({hsrA=bj2x_q!HQ$%AFgVHbqDyJMI^ z&-5dhm!TfrTPblqr-%f9?)TaqZHz-@;0VJ)%#_=#G16~Q%cF0k%E<$_;cB_$jzA?> zd`wpwjM}Oji9iR^BV=W2Lp0oFj>)l!7)^{2`C%7|ITcfie={$PRT@ouxKsNLjZude z?r5OuV1{DEdQT^M<9oR*t1YKgdoWs|p4Fj~fD!!C(;BjNzLx{)>IfLbIyt26GM(n^ z(CJNigDL5Ce8vf~ru&IEA@u~NY$H$vk~o?1b+J2UBjJ(4c%8yk58l5DW?#JtNZgBj zz44|O98RlVLs#8a>~P4S>*By_qmS62nr5Smrms|Y5X36AqC`y>=%PLM4(3YE%*Q(1 zvS?9x+n~@|9}ktQ-J&+bMs2RtkbE^8k}A?r)z&T#6)UO>3ZzzIfolUgz9g`wnuugI zk@PfOZp$?)*KFDgLKt0jga4Gv7scXjrZ54$Q_=z<8~R$x`Q1*%)D9zvR=!gsN*aYN zs#V}=>MT8htu!X9JMi`A3O6tO-NyGjkGW6Y?0EA0?4~ns7qP*cQ~l8pUz3rtVfG@f z`ZgP6Sw_nMfgk#W&{QkONK`JwINI%=wD$xgi564I5mBFh7)wM1f!AB*$kM$oO7UYZa;2 zfvtG-G_It*Wt3P2CG=F;zYrDp+ea zB~Ks6e~{8cKG@lp47&6`t6_*?EH|FW%$R|;(2O6Ul~z??cjQM|T^wkkEgnYv+e%^~ z+d+`Hc(ayhsP$nhyY!9kunn@8F`r6*ceU+WLkw)N&N<0q_8!^0##TF+Z+1cSDSv*Z zirpTqp(e)s13KvfW*ABp5(=DIBfK;{RysWD z&I4M?L|Dv(TfhqMvG!Tu+piOiU<8xP4Xqed4os~uWdf@*$*!eP! z1-K-a6Qp6-(|@^r0;Hj0Cp4EKjjMpRvxjltR^&Zsr@k(zobvCpq5_%k<@gsSY%V}= zPIO8W6{8J6oE6ZF=fTcR?3JdmyXY1dNIx7$^5AfnG2yBep}d69Z78wD(BV?Yve*$Q zd#KbT6y-1mYwvjTS|{t!N=MPWg+Dg@HDNbW_ z?^nbN2T@XDLHxEQMB)w$6WRbnrO3{g9DXJgUQJ|I`;nnhsZHY>ikjCs#A2NAZfb(#QE|jaW~G4qmZ5bpz+KYIz|ud-?S@; z&a-&FD!SZn<*|H~MRDh+MO^GG)f=jyZw%i>3! z7kB+%ug_t- zo<3Z9O9#RWJUFC!Gj{$=SCqB<{a(fV2YSJ$1w8Uw7-2{{;|bAxVb-)Sqx55x;Xs~c z&s@qk|b+nylpF@%)MkKC)s;1Cm(N-rSoSkC7-xVWXAc!^liq< zAat8vE+%mltv1HLFxR0aEwy>(6>fd|_T4NuVc{!p}P6X;{8+x2`sF;`yKyMysXXST+d8{KQJ1m}I54+t&VGt}vR4f}*Zny)x1p zn^tZbzx}4A>hS0+kZKx{fKN4_h7tY~wd|vuTSEVf<^>bI5e8&-pw|%e!?tDM(dt2O zgp9d6a3S+EbsXIJaJBd{bC`hHCo%3IZi;u3)9cIx-y{_$>Bb-K(~px#J@$t2y*GQ; zIs4i^>i&_%WRh> z&#<=7A*haFRX+WIn5WvG!MoB~q??9%n+0~8C-&AF#&iL(A!z>wL}M_6Sw0*; zn8(i1Thjr7hAD+L3~Cr5G7o7GO}3tA8O`w|*@t;>}Eqr)l zkWDIXVia{7vzgBSyJf27z?oQM))bs_(DKq{^u?8T()dD&V#kaf_ZPf7K0yHsn#=n6 z)z(*e1B9+ETnzHAneMT58B+Tr)0Ovx^9g=gtg9`fKh5ghVvU&e6W@XE9fW%+6z99= zoGP&++lNz#Y9DZyYl9R<8VR9nM42f0n3n$i@p%9!`fWmRfhVNbQGFC`$1~OV?0HDp zDSh$y`N>yhivA(j-AYBG!KLqyu8n4*h1w}HQf+Vne_Lt~LvY?@XB9dY)cwLY@@Y82>bKu!gY8HETt z4D08?XA&BLU7!-b-|jvMH*o{boTZ%pnRU5-|TmE zK2|U6N7}ZmwW^0c>Vp(^Df_8c9#fH|vmqBdG?a1m8zBq++B97?ap$8etvP4<%*yUTYDW3adL?HkNKa(7o60T|42!ykA*;O zZYcieY2e#<51uW@^eX()7VzD6b^|3V07)qV(}e!k73zIwT=**bFa zS#eK(Dpo9Zad(OLDoQ=VsdHa!Xf&2SP6HcG*xU)q?tw>k%jP5$ZIs#`M7Oj21=YUB zB_V5+!6YvSkf4Rlpl~&+Tfvaz*4-H}w;XD>-tuxQ==F7VwF?(S3%f|O)R&@w;s1Ag zPo+r6YyHSc`sVdA^h)}9OGU|@N=yt8NJ+(R#qH_8?___7%KkLGybMLy1jiDS_7gbl z7z+37M4C5|SFm0a+ipZIVG)mrXiVLbz#ydaO}4_R^ZQ$Z?|3Gz^M;i6{}sPC9Atm0 zJrLvCABpLF>u~?;w3LN8s!9;tTC&$3tyR znq;Bo*U^6~^zL_=8{FR9@vfQ@cdfX#40HFux{l%SyK6Uubh(b6jk(T}KdGI+sOfxU z{PNm^h~QtDKH>=W#UaS^6`zEsA6Wy-edA#PNOx3dPkT*)n8OT1)jA>U4&B-PL6z=p zbh=coreJ?C&H!It!vk@nrkXh5Ywg|qCgni{CdxU$?-%X0KAAbgO@q2^o&s1)yyEOh zi(x@98Vp9Gxa@foO8nAVz9|0<2I|p*<}w7^wuPZe8s(zc-%D`V8@0oPyJn*$i-DGv zBrQOMLW#^#3l92S$*Q*560;A^v{e+miZ8=Tl$XJPtpg*LJ~zEmJ&{6FH$C{6E7xxI z;G66DHRC%f5jJ`=AnP;B{NbVl^xl^Pl+tU6guWa-zVXx+i0+YO8sa(-=A z+?K3v#%$v1#BM`n+cm~Ee%OJ>ycL_oMG?{?^L^86&cgm!EnB=MOda0Fc{#H;$J`YYM0nAZN}9d%B6+6%g2i1dao?Wi9~KQ=OE6l>7QnI1s6@&i=M(^L00MR zz!yw52{dAT*I-cFko)qAvx)^Lt_iueE8Rt-wxlkL2sS(4tq_TY)NoR~*Vb!oh6zT~ zCD`UkHe+{z9jroA5SPpZjDRB+;m_ts;y`PxY+yFz>ZrK8R1w{o=}$n=)N_YK*kmK| zCU?G{KeQ8Zv1s~M+bNKBpHY4ki2x+HH9|Cf878dDtR7cm-B5nzs63eYe^}6jDM&mV zC}l>l;u}tqQa?tuzjBGd=Ims`(qwHf<0XK=jFC^oE>;r3Yqy}mr(kRj)z<#@B#-9c zdA+5uCD{LgiJ#oQUAqCvOWrV)l&!>UYR_HOzJ$sv-5)I|gx#_eX2B2B$5Aip#@V1E zgRl(}D31l+{e}7mZku};{tqKXZ_>>a-kQ6rjO7(!sz2pHtFqO%~JFlzPQ*t%X)% ztL@`g2b}-{@(;V{{3LMsPwqe)An*2f#aX;;59KD1V)0tF6aD5mbHNI#uxaKn7%%Umf4q$rxO>GSzg->#)|v?DKhnFRCWh z&GLRjqoNWOWYL<;tV<_k+}9N5f7Eaia+?!a>wGg$fU%lDM;#a=nj!M60QEsBGN_W6bM_JK3>>~)IOq;go98vC#|CEh)zV=s zPs88A!q9R~Rd;`pd6DZY26@-0>W66ra|V~+q`M4KG6eNhYwNkBL9+gZ*8VQoxKW^m zMya&D6#BzPwmEE1uP-XeM_g7KXI|fykJpuIk+;75lp5>9R7HS;PK0g3b0Q*9xM75g z9`7D1h^>|aqJ5R!HPwVh+F^Z_;n1oy>X-Joz0L*K_Mz5bhTQb=+&{&HdNzf3H1Mj5d8G#i=4(?2@ThnQ4F{t*6I{k7x$OzW|Ea7@7~Dpg1wDM;=2k^CaL> z8NE5E``~Ptqzkt5-3cQ*ESLFJ8$l>1|F6t|%N|^)P%4c>4$<`sD1>LmNg}GQPC>_X z_G;s$;_3n!WJ?5SA_=7o5X!DoU4>p=QmE?h1`GK^_;VNX`iv{_Bo~04ThejBovsvU zJX-1)NC%O8ucLFG1fea~oX7jMFqBW3QB*l2*zAXq&KGp-m^}`Q;aWmY&)vhk-CM=s@;9wqBKd%5>GBXFc7}?NKAzFdRs$1Y^ zg67bCQ2e}WC{!d<+lKeg7mEP^s%Z(b%knvm3RgQzr?yQ^5HQ}9h4Z?bU0~zgeppCh zfhHp>P50*df=7vIB!amceTpYWd&1|SX72<-=}mK%G$-JssA_gh0pz{dDYI*RO%?YF zmKxs7imb+vNFsZvU8c+A1sSojpvty>8&{g9Tz=gz_XoAf!K*88_V`fgbuOMGi`3#B z4w`c=g)N(*IuG$?e@W|=m>vQA| zJXY~Ev2T$}EYSkHRvFwPCu*2X0Af8*;%Ey+FrAV1guRxWni(-9rE$Tdp6dNj!V0}4xj<7}`3 z;1ddXCs{V17BRylg|T*#?1!dA7a#Opu*uM^c{5`>9A>nxw z#s2@c^1rb^!~fq;KeM8a^0>J3)=@D`!|pfTehSUT5mTc$oN!3pbs$?u-DC>YM+TSQ5D^pxc6pA|4RAS{T%^Gn{k$oTnc;xNEksEy!0EW(i z2)ngVrw_0KQDWj>tFqdbdhc;4!$8Hc?!|c=0);O13(g;;ttM_BN|(oIykRPi#2i%1NMpxmsV z*{IWzU@g8y-VnFncfs8cu}e{P8>+%fd}TbrjlJ7`*Xvg5aaKp$RFjiel3&=>^Xut# zS+}piq5r;j5mif%0#Ixv@r^`vJO1S^JdXWk8bXz6hYoKp-FOHz3(~*HSt?&(;inME zI_F=d*6t{h9*Uc9r=_R>kWBd1JCbGsEv8p$#6RZ9ST$Oo~!{V(K{_EJ+ttDto zbx*&sH3%yq2D8>|L(_`sW8ua&t5&t)movZR88%x=*xutqmp-URZE#~OcYb2 zHQWnhd+vE@nb{m%Cprp4{I)$DajY#zt4cljO}NL}g3=x$N8=tGpZs~~^3tnsXS!^+ zM(sA~95ws2?Vy$#Nj(A9kr{=ye}wH728cO# z7Z|#957*_>P?aXZ>6(BXS?)P3?Fs!BGk7k>Q7Mkvy*2*JJu-!qJqMa>KLyJAUzjRvvqhO?rNZR+c2o5*hBXJLIv-Z(`EjQjT3kx2&D{oTU zpyaw^THA1?ADN7tUJk)2*RejP`ix3b-N1aRt1gtVMeH?rXW%2}w}Y)^eWNrG5GCat z;|bY<>XYOO6~SSw9ki_p^#rFDRCmYmv|QPa)PN7|FxawRi!8%mC&Ap=AKwwF2KA#^ zSWE9yHQMa7;ME_oHtBO<2K7;qf1WJ; z{oeq?-nU3)`@Ia!%dvf`wYO>xAu5fjhHqmn;63eomHTsKK!~ZA0BFA zFW&hKh7h{So4JmyxihC6QN8jiQ5TcW?60!J0_sNfg0yvQet-30qlZ94v20dZxyn}x z9u#oNs4qc5QO4e8)J2*F(>dxFG1b zWjisrFn3<0Cp$)%Q!Pbi;5;17(Tvx-0)3F!m)x>D9a&kCh+nuz^V)E^`yK5D7$?^v zM|t3jjx-ehJ=FjtAWoK+LS`wJrmiBqEaBE|o)WzM<-Qc53GEE*aa2R4s6rreITn1u zB@)`+4IC}-jY-HF4r^$+(NI9^lXhQhw()wR!2<;fNwCPy>`MVA`9|Uo;`ciD)m*kC zfel7v1yzQaRb3!%x?NpSH8PJza-I$Ri?-7-wj7{W3%22dOpCpFpRqq=m;dE(bpfUf z(VMf!h+`8Fw%`o?X5kte4z@X=egB6KISyPI4wsuW9YLCo&vhjc#lE3-*qNDhh0pPV zs8e&9j4z*D^Eab2T3``$F4$!78M7s-8>AbHk)U;8r{du>wwS2;Gwr{Sz+toeX0!Ap zXcBFFx-wNO<)}ATh~gIOit*&)C((yMh%YX7Qc^8j?g5!8%ztfB^(Ldl#wPDZFKs%* zy%+c*Z1M(cOLD>TWhFfUKUrzt(;|{*>A(1>2jb1gD}ampg*fFpOhmFTLq4aWl6`Sr z_ozE?)cP>~k1(ik-LgEYR-daS7|`oE&Zss2V}4Zi;nxmIpUXEw@5xNH0c==iJ{QqP z&j=T$*I@-88jyVpYRS+Dgb!JV$~wRVlcjyLcVvxfX;P7#GFg3plgOLI_!@B*i8CP; zj*$hA5%Ey1E*A5S(7Nxzs%mY6S%n~RXz#a5CI!zpI!jqh037Osqn}z;q^i-3MPlS9 zCc(S#(q>_}T!@pX3AEmvHJ_mPieBwLv9_gHK{XXw$vog!RzeqF(u0*EAoX#3y~If- zZH6q`6i&S{{>MHGS>xd*mNHvVVbfv${yxHkqYW9x_(q1|C9Mm%Y4g^g|H;fXcZEn1 z^6{|#r7@zi*f>n?XgfyxvTyf~j=*A4;QjpXz;m5yirA_YDbQAEL~a!Y)(85`m<3M6 zcLP_l@qPYhi!^F0-v8M_&pyg)YDp_W8iKRqalMkC!($jPMJ-6RT_y;EAX~5UB*x1O z9oN>sR>62(ola*L6%of@I1@?sY-2WzyN-8RH)2@h%(t@seEtoc`yJH7-uSY3?jx9dq}I@b8dDaH~=Jm#@}9dS`0fWv+i$q=2^V8 zev3P*M!eyodh5wvyp;7RI{s>Yjah2!e7Pv}uIqwgd!?#bRf>xdlt_|nS*fx&m{yZ< zedh;PWVW`R8t})eMb-NT&d(J*ay}@Wt30;}%Y)8J$SSB9PZx>D)ZzZ{rB7CNzpDz% zyxqvIk*+=5a(OnK3%YD_%<{p?_MtThix`uS5^bwK3#%@M#n>42umi|6Q;Y05juKvv zkl-n{k1^rHklCg4!-ROr@lWPbN$QcQ$->N3DK`#(*UOGN<4WXvNoyPjf&50Q=l$!S za3Pbv?>wKaBp%Rkxy#atm9sWPk{m3wJ%}|>x3A}ckDqRLm$zrZc&x~H8QU;yYr5_P zkv5@ptjOOu2fx`j^*nVW9m1%r>9#k1(ZtQbi-_Migqd&RpKIK~G6!KrDtxXS{Fd4@ zu?w}S9MT)LY|r4M01bN}3(a5If970wB%Z5+_voF{j8W6{zrnPAxtEL051WmaBg8xR zu};F3(D}Z)U%F_)qv`4N{QAZ}^^HW4`pPhd*^z-OIXHJ3h`g*WubSy+?W8B{!*yQu zs!b;22emavfD9|v6F{a#^04?Vgy<2qTDSASARa3Xoeiog;cw9VilIdX9giF&=g~zJ z+CEVdGcRrwoN9mD4KQj9eYdVAs_k_}{+hMsspDTRlg6O~PBu4hKNY8cm-%KI6lkA zA8j#y>CIzj3Nz5Mya9S_gd#CsKI|vbWUOMC%6(sS z{>a`UWF_Q$;^ldkzkeSd&iJ@}?fkqP9**SLKfAv0*t#{6zJ%&|LlQ`utO_DXyY`LC zqX`pe=ItqCnX>9i7SYyx@>;6v=ortzXRD4CN-AQwj}Tnj5%pYTKAR2dG{Bnxzs&=D z$$;v2TF>{5z`;H-#tL(WU&pQ#_Rr`0 z2ypdV6Ju7a|TEIbmQ51d?WC2NG2dQGVEH&sk1V zH%eFmgc(vg8> zLsg=ZP4wn(gPzN`C!NT|5KNF~7T!~;P!giJ@;b!lt)zU_y#63q6CRPKJovy3J z#v>O!30rUcc~i#IEqhB)xMNi{p;2Cy1X01f4xBCn4@&eP0OqB1=6&$DoCQ3d4<^o! z{l~`-(|=zDuWSFWMDmlT!UJ)fI&w;S)@ypg!-dsxa&E^Yrgo`BOSoFFCp!bmjwob) zHBJN3J+tiIU+l9&;lb++1zmv@ks7P5E-x#sBS0Nvl86{8&2!s2Zs#h;>0H};$lbBLT{QYd$vBI1aT@zqr%dQ{lA@L|4A%X(fLs` zO$oWO#GhCSJi;W;c6?6H)2?FHxJq&DD2+*{^$LZ5?ON2<+9*nE`5)=RGm$?gG0%o= zHd8E(G7-HIuq`F3GUoZSwOY>3vv*r5q(H8+LJFm89dDePfS+OB*chT4|8FOdu8wCY zRqJgh&b{)f9uv~l6U9s_qI8$4W8Jc~l^w5|?sk1i+&;`WH8Yrh9#%xuy0(;|B8_j7 zL;4tjj%rnhGHYT6&D<4?2~1*dA#LC(md5rFj{gJg0M1!ZDyhhaIpiBi;Z)3#w3%Lz zq|=Dl7hY$BFfH_fSXmAayGl#hOQnnQK`*(q%H%l>_$&N297Mb&lPQ6g{|k zd#;LOWOVdF2mg@G1EG&u%=ebqO`i#=C~UZ}m)}iwZIF;k`(Z5<7l;^dN2>=^f=9@1 zQ7KysB&U*T3c=2*L7mji z60YJs{ryj*9~KoW4~^^SLrikWFfLGFVyY=^krbL3NeipjLBw}!JXMvTzweXUJx6(y zs`5tt2?9uH_D5UhKy`4^+bSE`qRtEVI&Jwkrc#KtcO~Bkqu48|*81psjF9g0X!cOj z?!Jsoda857Qy})@8NAq6pAPWUo9Wxv=QM<)at=jWQBzXyaTQ8cC{~V!*+$OEbPv90 z`O8T{#9#O;MG|1DtF+*EugdaT?{>k7&NVjc!+wB4Zx$^cJ|3Qpt!4X>P6W!jix2RE zEwUTEmse0nz7=wja;2xU0fGsct85wd4%Bkts;$k&uA&fpf+qQ;BO)0i-p|HL5*+#+ z?L@8%+2-i79PWV!4!%a3>x1sa-5xBGo2y;>oSMLSM2sSaAmAuR{%`hJnczE=Ugn?g z)LhEJF%X2pSm~J{+JO3eg)J&%vS!M#g_*6n;_OeBDUX-Mr4i-G5Z7~<*uQaOCDQc> zV2RabzST^PJ&0H(Jkg|JQGN$j`hqW@JN^c9=sH zqokX~r4Z$GkM6WQ=IAX^yrfAAS=r`Y^G|<6i+g`LG2}sTrk$&2+M6CN8InTQjL8(! z2+Q0lFo2gb;}kPv6f(D|cw|>Lo$N%SFiqv7qi}nv$t&7J0hD#Fob6-YuiWV3s&9fZCqRgQtuj%Eouq61aR#@7VM#m)bv;VFm^7bIboNw5{ z(F4Zuwvs@?AW?u<6sCKaJ>_0$z1oyKH{m++g++hvSS1kKUw|QPame!^C%5z z<#E2-=4QR-b?u4|8Jc9o)4*nfD4z1l|Be3&Vk}Dj5BS?Y{oiLO0RLP2JyF(E{5!7y z&jdoO|9|-J-|anL5#q@IuCQ4ZP4XOMcdw4D&Oobk0*7$cQOk+ddj9JjyV*Gr&BNhK zv*jP*P?w!Ey~2b}kAf)L=E7d0k(?FBTYFuAsuxLrNY|OD{$oq>rS+U)KJEj@_$`F& zUA&lL-i;{z!2V)A?$~3PEGr4KX7bTXlv9{)Sbvjou)vtkvT~EY?=V=HTE?>jmt#(y zX?t;$Mqj^L?-b{w#wt~1jLL1;_3#u7+ilKj(m_$`=yLIOr5$FdCt@eAw*LFpL{Q*e zWSmlEeRP<)D@el3N}dklsVVb2?eDvd!;WCi#+MtQ2!#7!hevkWLLyV0*@*}7>-=kJ z!Z$==PmK(%rQRPS_cgXH60uLtVX!l+eJ5(WZniRaB^7xMT69dBhlnf4U*4bZ`fr?g zcrMOhA+(#lRfPqcFk38(c3%{na%;7jY5=^+qop1N?Mh{jX6zh@tJIEWqo|3KVCXzsVT=+xN)qO$wp?(JOAl%>*>h z$l14u+D#*s9j~ro@!>5x&+x|wTFKj=o3xAvKAxeGTT4shD`fcfF(0DMS#LGYY5gJz{P}zt2e>x!5 zFYZ6R6}`MqXCx6k%6o29NqhLOT>qng|Ixqy=-+?z??3wYAN~7}{{2V){-b~Y(ZB!b z-+%P)Kl=9{{rivp{YU@)qksR=zyIjpfAsG^`u89G`+uc>%DHfS66IerPmHS0vhU|3 zky=WXb7~rOb-9L;31a*d`0Vj9(ho&naFa->UGwzJE7r;wx-|9n!C(5Rp^9sRk(5^0#=&7;RS&KHs8Q zJ_54yimb+s0IjYwX6y)$IAF5j+=$N#96!v&{#(kL3VMsTp%OSs0LGjUjKr$pN;k>U zm!-Sx{5%VIH3{hKF8)A4yr{E_#FwB9)5FGOV>MFyn;_tMrC-FvL`Qw4wxqAw$eYA(#)F8OUw&4?-9&=4I#~|AKcfLF>b3tRBOx? zyV>hIgl*vI1ugkf?p2HJsJ-TU=j}hPNG}8g$v>$*H{Utl^ALg%Eibp*DE>P2LLVjsIH@4vJ;|U zL9kNw$F^hR|EW}Fxby$?`@9W=$a&j?cJ{i3XufDgw%xoaY6~MT+=m!<;=Uo&-Mrz2 z3)8+!;spZM@`i9!FIrl|iP=M+NhXk>i6qm0WOZ}dhPGy4UW&RtEa6w}zeXab6 z*aMSlzqWxR?_UM_=i15-d=-Db|H+g+A!RUgOKgI-b!ptpxQQ_M4`c(jQbQ78l2=xl zz_NqHHYo>OQL4r41}m@ZatpEhE_f+PnOaMEX`J}7Aq`szVwKL4+K&3VDe@qA28YcA z>d*%K#{$uvMO*5hA|A;srOV0-)hH}B1vlhd2`yST%t;IzhBRViNVVwVM zyp=^6h7uQAQ3{4ZlNCV}_z23zhe*JfX}aM446$j{J5HrIKr#}$Eme&Si#yYLZd;gk z6i0J#h+^{9pcrMk1d$jtO*zrX*a$ro@s?fhF9|30n5+xnZXp$kjcvGn@3#Vyic(qK zU7&XJOhSJrU#Ws8F^Bqe=!PPX4ThjZL?_GUsC%(js2_t*qk^JqSVF!<60UQPSM47P zBfiq5I!%&U4d_4EfmRTWUi5lIoP0@NT|pjpJw73erCi8jQprn!9NGGBK-c6z8K7bAn{x+L+qG;cDdc3 zDCDBh!Qrne_5$qSX3tbL`B^(%Z9Ht{+bn^pho7hxPMg|v*WE2 z%GdaFZ>!cnp7jne*OPBHv0i}qqLqoBxp%}4%oy>5)OTW&T1Yp|CF#Weuz&3}$Tl?4{u^}m zVfk2y$;tYr3;B5+SVJ?-+2rNnB5DV{}7*E68S1{$OWbnlt0ouH0XBe-T$NlK%-?AJOL`nZ ziay*(=+6XLQfZ{G*wPdG^EQ^967^olW5rW7?e2s9nM)GQd`mLSiW!LN@WB8s5VD7M3jrWlw?b zVmsW@P?T30aDz?>eK?_^a#>S3K}v(YOmv~-EA1)x*QC(f3;P0zL9vDe=5#Ep2L@lPd zZF5!faKb)~sy5ly%`v+Sga?AHYr93FWcA5f239|K!)YS2Sxw{m;G{f;-MMxV`1~Pj zKqZ}KcW#S-+jA4+R~pqebogZ_>KU2Abf$Y91C8ZG_NwFux5CtmQk-8&wR>oC?SeqmQK;n3Dha+GS^o*NTHy) zsxE|=Haol63acD{>449=2G!wLBEa>DkP@%hALNCYxFt4ipXV-qmTGsa2}WZ8VeMe_ zTj~_7OEi(fOzS^5r1~33-G1`Np0v&!;h;Blw|gGd!5?qY*66Q~o;YY?Db=`&|Ka-W zS(Hjr;=e`Uzf-{WUMu+mza#&VM=y5Jv(_G8S(nr%g?Fpe?crkpI+Wg_?!cNcWx`U! zVB~ zWK3P2gx*dcnCFb3$FAK2ub;CnAN8kDan|PDS|c{qqx46tKD%0-hcUM_{Y4J-Pf-qF zYCLfo3x?~iW_)gP$g(uzDi;ff0jCh#Xn9FrfH6THk8$4VNPGoTBI)P!KFHGG?+uuh z&2#IO`4lil3qOChzObmX{@{9JYJyOv9AHfwpzr7SjDs<&h(3%vTM@~FW-R4 zg?hK}5C~T%6lL|OPzyLC#NRc#0cmH3TpIE*^kOXElEGv}rOR6nz8VO$YU%Fs6wZ!Q z1Ueh|qpqMspcN2p;<~|aO^dVF)}s;z&7hBHg zaKTwp;)|)7Ng2!N=0>-HiL3ld_c%*;H)JXLM9oway^$|JIl65SZ6C!A%OQV;N4pv^ zgFot`cKX89>5oLk!zpgmWoLJ!jg|!WU`}{>pj!)6;Nj*M6eb_pFLUuQSq{PI;#JEz zApW;vn66T*f)B8?E=JgWfB91OIGUjX?JiY!V?VeY8<~zeM2u7HD_(bW5g(K0bw3;* z>DJ!doqv8gz!|>{bPMKv>sH@fohCJ}xHx<~eITsN(Js9NV#(Fz$7a04z&@DPt9``R z5ZF+HULF(F9WTi{z@$bYXvTfzb@KmdtS_n_IWe@6an^+m@ASvO`!?y)iX(>9#N@Ih zB*$k}L4yx0KZTt1gPu`0G6^i+o7ARK5mS}aWS%otDxo7I8F#IayvxIa*}X#1)I?iB`e;*I zMZRp?fjT8_-OF#x(@$eOV1lLWWh)5bbZ^wB9YyqiJtJtao)I2(W^2ske!(5eQi_(c zJwzNApH|ytxt@mni_(KynM7kN#0iohdw{{3shqf&V z?(Bv-Uag%hQGt+#fvbA|K7Yhs*(~fF>o>?0>a-nSL!&zcn3$Gv9S^u)Fg>*sQtc16 z`eVkWWIlMnN7V!g)zTTcsDXzX;1S;u3!FJLV?Mw^nz4t3Jh-%qJ|cpQ;Q@$5pM~t! zXe_?hbVgv)=^ufp8E{3W2`{JCUSaE3^ig4^gISyu=g5LU8rt);sDR=A(=h)6v7fLD z4wpH!O|QkFfjfk&t`cjFPln9JR#j{A>^pu|J$U$flv)E-5zEP_Dy%7A*&tjR>h6*{ zgqZgQ5CZaDZi5pB;ZwTE*aQg_4#Y{8qh7BJm2=8aOoB{dtUQQ<;Ig-6XVa4j(b5K^CDsQ<%&KmhLg&UjOy;^w9^i*q=5a-pO zQ}o$h$C?HeT3cJU%&Z@bj6ypa{7C{J;U_6JMWn{Dr&hMG^pCl^FN+vrAdY^c1a%r- zf>~&&=M&6^X(^#HyiuxWgY|LL$-&Q_tUM+=l`_#jKe@+r?x8SRN5Wba4&-EDK#nH@ zUPTugVs-Qy=p!Y>=wZF1t15i%05UY3Pav^TduaX&z#!hP6OUN{gET{Z76HvxX^yyc z<~vV^ZDHm;bl-~cZiCm_jtmHG)wo|#6$~}byZT;eDw}2~F4pjFDE54&+8sMnf22cY zubn!$<58LJ3!+52$Y9pO=g3xZb8TOO2u#*|EOJMd^D!@LD0Aivh$87B617X_vop1P zy!1I%KwGcN>&R+TDyt8jSIrtaB5blsKN6F zF2D#v0krnKF`I906>DjEXY;;*m=#n7Sky4bB5^1qQXR~mv6d=5fhle6L6l@=JPF(e z;2_NQQIb4|5@&>GudK@>_Yb<(UvFD(dN6!}hL+_gf~(^-k#g>bHaG@rwFHvYB#TLV z5F$q@N&?I^NJ8e4wbGXSsft$W06AY)7&IYM@#5<`Ly^H&hgGab&Fwf-UK_RH@E6IHRhEpLA!rxzN=3%RnJ zs9vHoca>d_IeZP0PRU13iReat;6@Hr(GuKeN*4qDW^`%QTXH;J_f~fF;ALn=r|<|I z4jop*bhTGAZ-0Q}Ob1Ilc#_gF8)|_?5+hJGPU3bSc>8cTGtmzyNz?`+vgV@RnBSAQ zZ(88LDrY{mJNjiNpS{~}jyUnj4mnzXFhgVHSb_TWp_mmZ$1$o3^$yQFQtUEem6E{S zx*|eq@gbotuj<>dx;Yyg_9uO<{=jW|>YpM8vb)x7eeg?FWO_i#u zwLr1Ny}a|i2zt@RoXh$I+s`--eBWHou*iXcURB;blw|en5B(v4@wx@a^|oi1v3*~Z zHLDd}7{Z24E%copTP$Fl%jJWl@I#OfAB5vXi;4QIVxQuN zTmKt`E;xy&-c*uB3!{wNM8C~>a#XfNKoA2(^?b?tvfK7)YvOggrTkYr3I5_R83H3( zvk6HoWviZ3Z?>Vw;>=px&a!3rE3K5y@kILGuZJ<`eSrq7CVk&;yxLg{neA~CQ1S|2 zaJ4#a>w7J_ziNGn`BQ(UU^h=(2YW3uWaD5agTe91CrL5ThRM~L?t`Io+$?zy3z=b2|ECWJ6)T!+nB5^hBR$>V26ry_ZRG#N>wwWw-o-VV^kZA z>58m?FRHagLF!@q^L}+{Tgm-+jTJ7qCW!1=Nv2WYOT9bYT9GnLBwx5X7KeqV>Dk+! z)T>GW0krW5g@eSu048!IN&{I6;e#njmW@I~DOEw#?s+HYUjHeqn?--j2DYxMUh2)9 zQ!rtnak{Obv2~BpS{6!kwV|<(ws=XygDA)lGc*}-EsR;E{ekI?!&`kau{V+AGf|hK zy z+1SeH77$7`d-b~R@mIAWgh{pa3+vGzhhKV9`B6TMCQ|&)Gy!(Mb-}TSXD=Dd>??%R zFIZQtyKA(Yg3?ERqlHuzJY-YPXTT0@K<8rANNlrwLjwlhz=;Xey=mrYd^!J}*RV_!CZ^!xV0xr=iD?3R z`2x_b?MClLF+Rvr_c`v>ayzjh9pu?clhFzHnJb}i z#wq#fvv4B>NZ|*U5)SzoZhDz?&tYmH+!%4X>cTLRM*DTP$qVc#n)gGMwcXK;X~m>= z93w`UvgnC34dt=!sj?AEW1})x>0e@8J$UvD)xP#!Ow1@(uv>dVw4?<%3Sp@!RRr_o zm6FSgy6qz-hkNub1;G%N4tbYlG8mDxLh2+ErAnVI`*?c0n?WJD&+=*nWN zT|j`9P}}G$*zdb}L9^dbTCFm+du4Q#yc0Ziu!%#a+l$Aok~#XO*1Z1g->LnTAG^yn z<!fr08hY_OzE zK-*dZl!GHQX&6M4dHddrB}r;o@?TMzKBtXb;%)369dUgWg-ej#v9ZpM=b{n6v$O9s z;>vx0xEP*T|Jnu&6yDq#6ut>~!$6zJcA=nWH)L2blA!8U@vJRPy#lvyCwTqhEfzjP zBjvG<%!S+=qmZVNPbvGe^4!cDsI4_6_y!h9?AtX52ZI|m<+L9!^cLlZJUhZhN#NsA z%zHXfC5Wwm(O6Da(svmRTXr4(T(O(e3R7{F1WX99349RzR3f3O2iB|)V7q)3;k4ZQ z*!Lxlu4P_*EHt*#TLjqf?Wa@cKJ?Nvqb<#HH9csZB~C&c)A%^&2g?!qsMqZ5U8Uzv zw_AJ{O&kv^h@p- zGb95Td#bEKG3a#=Q0h>So{oZzwq7zM8kK37?`^NAieQld@_v&i{%zu3my&G+0oiMM z5i|i1P0UWO{7RdRPaQs|d9DEt=$4sGEk-5!_W|M}DvcZr-UJF2(WOMy@{R`{xSjem zqJx)pB|^5oGq~$!AMpy=!pbGAkLkPE^79z;Uzi$)hhD$A3-6sxLrBx6{;SRZ4dFZ@QKpfjGSJ@&#Xnd?03V9 zH8$q9O&j0cUQ7Lr?)`<(ZC8ht{ZPTChifj)Jgo|_d5bi|YzYO?Rr7)R>j@jI zI%*oOJ?1b4)6qP3*#VPQ9l0vUO!Y-#h#Y=xS;>;yIMGH8#8ZX=IWn#@a}lXT_%*ZU zC4_>Ez>pmra$~<>bS&A_vO=d3{3X*xjl{&&LLj`YK-Y6q5d(&a&1eVFyOXyb41-H@ zPN&?mP^f|Id!c%LtvQ;F?$J5N8a@X@EDgf4Tj|FeAP8w)9_rzR!t^K@MP}nc4D}N@ z-9Pn;k6V7QnozP!e30l`wMX=?ArG!w#Pub81^pc{?K#EziN5NHD8AaVR7WS=;azgEXN@h(_Y&@iL2?we4Mm8>Ih$ne zaSc21DxI+!@_S>>G4gSDNtr!>Gy#`26u{wjRM^CPk}e@#gEpsP~UC{>|J)&oQ{8=`BeaY-g;5 z6l?(m7I8Cb*>>__O^rGZhpX~*7;Kf3d2<&>nzAoWqYNo}b_YlM=8N_P{mG)iZP5ZB z7NH2%H!@F#3&Gx)Jeq9{2&+e}Edr2@u4fY`|6DSp06w_Sy z)gb;Qdsbe6KhrRpEhtP>AEXECFNV;igYgMl0D#v99-p|HR($PN6Ds){gvZEGP_)F2 z#`pWJke?$n+|Jg%iNQW?%3crZ%rL9zs^!c||M`3fW{2a4Hd>0TFEHP^wT?vs z24~ohI=a0i` zDPr-WweC#Q+wugIQwu|bH(ouqnR9PO`TuOxQMKwbaI{1iVhzv0U$;T)Y9=0z^oNa$2%@aT)?RqKAuc+ofIG%nJ%1BPf&K0X+waf z;JcCPDx?5A7XI)H5L>pGDA^S=wH0WlWJbUL?$kSUV(%3dd)EKac7-$m#~H!_WvVJv zBiD)k^<>WKNfMt`<#_Wh-FVFMVQS6M7f~UZLfZ(J+-DMFKrBC$Zbf!ZCLg^EU$}U* zUf2%_uH7&k>lm6L5U#mw29#0>_9OZjJ&gORgs;c9viluW3P5_IgINvzW}%g(tT*FS z-0Q_@*t72d8uwbU6{3=WbTUt@Y~BGF^<7FwzYl)P1U9=<|HfjJXpKXw8J>NHI~IfNR( z5hG<|r@N7RcqDw8e)m15kPX$z!g*y@D3)a(q^Yls2~_$ub|eTzCIZbzW{IA z5;<$q=Ojo#Pf|^}X+fS)P3Z*|Z6j7^Uh4eRuLT!_x-RD;CaKM{Z+0RFUjbTWFa=n; zkLQMO7ShU|Zah}TNIuW8bLrxTZ8UCR1yq34RHrb^tCMNXqpnq!wl(`o@H1BfT{u{uVWn9}Bx4Gsz-sI*pbW@)U^^7%hAorQS-uUzAA;N!r|>5?)b zXt=0RzV2u)qxRh0J}>&xf})n^m{087odl}5Q*SI`21GdYjbT{`E<-uJ5U7RiQViA- zM&@1&QfgpTZ{9Y3A;C>?n>$|@2-IHU)HLHOh0W(z*cyqW@X)RmCu1g(uGSL_l8ts> z)7(cU?9x1xbwmh;CH+N|vV3gi1b6@=8N~sq(MB^;A7{2I3G3`X?RS4$bMiCt(L?=O zh6MVPdlmv20rCUWnP)5(1@;ZY;*LB^0rZZV+X~FqOxjyhr>6^agjxsp4DrYeaz)P} z|4t`9{~;U8BV_-WI(Yhve1`CH*RE`ua3OP~@HN`ZQfvQywD2gPk(uV6@^2W!Uq1#8 zZ{v;Cq+fFCA--cAP(sUUu9ar}kbc7)Kvc^F_;ter`c1vnAd3ZgwJd(s0t})+5EY1r z6ig^QP?h|?6}vLG{krhJnnR8}U|vhm>`>$w`l}W2auu;`zkE9}V`ByGlOeAyfF>yh zRUvY#_kAlt23hs1_xUOXHi!DyF+ufE(-{g~p>E3sfmCsT8ZC(Qk_cZQpJwUQmShrG z;FOx61yP5z1o4#(XX*BV%c;D3$#Belc9k?m!MV@XW89YuvQamd#ac@S__(8d-7upT zaS`uOa#oSSZx$h}!*elxozH=-!tu`ooX)`k;koQUXO416{XfgjL1KKxpnmi?P@(2T zP>NJNp%8i&yGaI*lWey|ER70@i_fs2v_q2ob-t1tBsd}g6OoX~81NJi;-KS~a3l^; zaz(O#cskUMeEcr=W72L>P2F^=0lAW zLFqRFOQQ%ovA*Wl%?cChMeGwq48s!wGj6H1=Fg#{8R~}{5QJ;g3eMr+29mA^Vuu&E z;hE*nfpQ|mlVL`i5t*Pt^Z#N;$G)4eM-MLjs5VOnMH2d|6_``{*bQ+f2+AkF2IFS> z?Q8}+nrC;*X_y;qpdT#44`&73_Isg4zE)tvf~Ua5lqxY60y^AGgeY>BgilTiEdWVC z+cuX?u^y496b+rsAHeH4E3?1YU37L8P&Vub6a@if(f2W^PUxe5n=?AqLVP;&$Cp<> z$b@NC7JbQvXjV25^oS0h_XF}31MxZ_027}RV$~F>ne;Z=TP_F4NY+-d8QAOZt~&?{ zryh@;9QQBle}OvT!DrC1mnP7{f{$R3D8Qb>y$@g>Gm!hq5P5C_8+g7R!+rN)C0+Y# zC9q=-Y;4(80oEc}Li$kI)VnVI7RzL(IG~lt28JEI*O8cwaVJjLi!xv~YZxY`HtN~6 zeg_X+5%I+5a30%X@6&0&;Zc5m7U#*#6_%J5jOb+M(P zK@0spd6*bCO1rAoJ`oNQGeKL7pbuW8qC=JiyYq^aka+g=054)6iTMSUvvo?$BaK5x zajBps-ivp5=s#GZ3apCa!Wb2fj=8XgwS{i!<+NS42PJ zCA)&Iq&=o>>k5kZs0(+Ud4@`SpDC4Q+4Bs~>_+IjJi^S>D!#pg>((pk-;gQN6HvDs`5$t6&KIK{Q$S_o~ zCBwjNp+P-q_6n6L>Zh@lDXtScXFOhr@JkxZO4CEOw z4lRLpAm&-cm|GFGrb#F@K-dxwdCv++Wa5th>>4Q;F_|u8J9~?}6(d-h2?ze_vnrU+ zKDq65Uosg$rG>YmRj9Kca!A)N_Osa6jAH;~E(>M543kyHIAemqAF-!MY6&}IUyx_OI_9t|+t85WT>wxvro3j>B7zHtW%%f49 zr8KNnGKgUkaGX1#KcP?;OQWCn}BC3sS;c1C;aD@``KZrO#j6Pn`T{Wd42pf`thH}yk)8-umt|W z1ydpYtt1g=iy0jp(8iE}NMdufIulOpR3BOe4SOl0f04tgu(jGy>}_tZI$&hWyBo*E3Fs}y;+cvc#faYAoCyz!IM9CIJnB4cs5paaJfsLHO`UD6vdvz9l_QWy4fuhX$@BSp6R9&SCtRB@&r`2J%HZQACwqgMd`TCDR2(B0PQe1@6O=+Xs36y#@4w4-p9nJ^B0lJlTn zGm+tHD%^rtbr$o>0ShE^n0c!d+s;~v{bga~Zt`Jm623wXe<3IbfZxRX{XhO;Z{nkQz9jFadQ2#4-CnbO{t_pNR3k}kD;aA5Lo+Hv zRRb&OuyiOk#!(s3D%@GNpR9j?G?bn2ndc}Esfc>4f68zr?&q~l9W8D{v})jE*rYHT zm-oz^=<1gsQbrzQSi~EjA2S67F~=1FN=_kh(RS`+T%H$!0?r=QtSl>VJ}>r~P@-Sh z-&7x|*$1T)J$Qa(;8oG~#kkh!9|Y;(TX2Mz3)f!|A=2=*K46j7Xmtqw7jBFsZi93wy3c~^)CtO4- z*E*q8X{ukH?rr{8jWCYDVDQ7zA`bUU3IQ_T?xF+FBiK+XOl=dp_(RckDpQ_mjvepH z(f~no0m%?nN8#x&RNgz}v6S4BE!UYTeOd68Rt0&RA0Kd>`KC!|3425TK=Y8F!>~e` z;;+ng1RLR@?1Yv}{u|^!7@VTBfMp`vDa+MYi-0lrqvTPo`a_!jzY_SmU{l@A)ZvU% z5FY6&07;GGS$X6^C%P+PelQ}MseSnMCW}p`xRf`8nRm6cV)W;$s<9=$Pe@p>9rs7Yg ziwfzBKY2@53D-XZp6?Dj_t0TpugiH~n>5(4EzO6Z#T_9^i@tmrC5K69Yj{@lajSj0?RTLFXFC(@1tJNOhGc9d~DiEp4D)`OWM^(651+0)xAVi@) zR{DQgB$x6-{mTs$h&kzP-Kuj+z^a&wsMQ(57w>R|1A-H1`SjtSmZ>!18W}bRO*Y%+ z5L@)+J|t&CNN>8r55bSpApCM73X~@^mJ1kp@`^#;B1FQhQN#lBZt0BH=C$sa(nO zaLKd8{R69F??=|a)8J)Vkg3Oy-(WJdd5hCghC{@UD9>81mXlI~K-$9TEYYo&VdP@7 zT8G77uu<9)m7O09Jt+*0kx5#H-`#$AnN7NMmiURM0;A^4t~nddYNt7jqeigo$KjC^ zi|h($&8k3^luzenb{kA(=1$uf$$W#tPgUbX@s>5%*%#gb5-&TqHx+&495g?^apcv1%4;_wscsY6Hwv5 zpi*&+CQ`lVLcQ9ki;bDmk6crH26si#n4xo3MD~?MG}sI^{ucm{KySYviB35_#x-)B z4}nE_7!=C!_>&eZCMfbTJwyAoe6PQ7iLZ&Xb|knc_LO?=EnAd zYUe{9ykqMLDt>r>Gp!hw-%<;jj(bIKte_|6*84hYChQOf=1@&<58-8gCDOJ=JUs7% zD0bw;e3)6g)ahW#M&M|VIB3ELVT)6#0gho+AFl7%LlloXeuFo63Z$fLQ8E}C++MKV zd})9)M(Uj1Gd1lgm)fD5YEQ$hTDjlkl2%;Kxfa#hlOrL<8(_C6-rmH zl#(jeBj&^`DIeq16E2 z9)*LQFCd-+EedKI{=`aQvf&z{of|!LCCL~#+G$SS@cbp--vTf>gSV5D_yPr<xMqMB;bPho98(Qb)h2}{>O8;a|3Z2zA< z+uHWOvO73ZEVcJLo@Uy^r3^zE>|bga$9KyX@9pD8$OOD5XyVx2Q7&$As-~Z3 z9}93sl^Y}CcgEquT~dg-)h<1eJN#uzE;Nuac;dWhTY-5VoHN4{hI#%2f@_&*f)!>w zD>9l9U5QkO=_q~HEQ=nVolhFqM`&hNVX(VDYbauZ0=zBF&|Vcv6SKRa_q!PioK{~x2y_~tD4A9&C8 zp&WTga*jGQcp2VY8HF9!j`#Ji}8fph#ZTz(CGO&K}@Y^a2bH z3|bTEY!A~jB{$GoGECnSfUdSj*l>2hGp{4}8a*^W_^KG+vMi+y7 zC1~MLBYOiu4r4cg`;>1Bz$=FN00wP&mwWHB_AMS~x#P-F++Ea_e%l!!_AK;)sJ>z; z-sm+%qM>EOCWnPznz)>^);b-wMzlAG8Hk9bKTPs{zU?v}-ZlsWv&}jmHbr67)kj_D zW-Wac8M%7X+;5CTcMQ&7m@PEKPN5IdU;PsR!b}BOErW4s0~C15@`@)kbEVVP3zIDPVQJ`yh!C*y{M=wGNT#I6SYygQ8u2c zKJap+2(3c+H$si23|{HPYVurG>McISjyfH^$fP(^?8Xq@G{#|BCipz8)qt|14@JhB zJS*$@((~Rq_T+TcpIvqceQ#Pa5gWd8a*NzyPr4gQ*p-hO|`9kazU&a z)%$eV>!J8QmSsu>llk$nwO;8o#ZCmF5{FuP;UMJ`BmR<#Wtq-;G-^4hpOt9zZw{2( zmbu~i=ppD5(+*6NTMFoMaGv=f%@_P00KH=jd}s_^vaBE_OKVT*3uq*q_7|=-0OIm3 zpyE+$B9E}F#_%R@9ol>j{0$-0=7im|a!m>$F)tHXJe86!@|DlGKR0n{pg}_KjIwp4 zwXGwt;)oV|-lsjlG2;8}Rw2coe0Kiy{ON{*KSi`V^jl9sRS@#;#93JAZcm^hEXf*{SNiW0`+8d3s7b*o-ZTx@SzbZ1b17hQNCe zWvXMX^&}3GO`sB>@$9b5c?;Xm*|84-*bTT8ctw7<5+(cH@S?YL25sj9jaC<)fh?%cBBsy7GEOrVo<4t}oqm$9ct*=??UV1Wu_quUvHYHw0%Ue_PfL_HZM95E{`KVB zi+4Y~y?FELRB;rK5UzXM~lM$J;{ zRB59cu>rE{`fUNKi%<&`T@gyxcOW@(V+YtquAvX|RYqmSlP@xn+^N1t8fN;QZ1prJ z7vEizmt5Y3D`#TG%r;5|HD$(>5(MIb!PT_UCG0~Xq~GLf#%9~PthN~IgJjs z=QVzsx@EF1bd3w`Fo0RpaA^6$WNnjIXfg>Tl^n4T)gO+|$kB(nRR84FTpmry<*LYd z!Yd_{an9 zw8iv+13QZlLUi>{=PA2CmnnUE22H?~;!oph_mfI;^z>*-u8kyg=Jwm(=cPCorELh9 z`JWt-6H!8o5}K2YQ4RQ}8O<7#2u{6VXE8w-Bq|VtL}(LaV{w=#3^YPnYoGI6lnFC1 z$8d{xbR?@P1VC(=F+M7TCsCRSu~O9K%lL6JPawJ{|FLG|@}*($4#0BC<(+q+Y>j1( zC-KR}ivA>W^6HCg-K^7JXwbZzCzM_}v196$usykI^ryOHN-l{KbnM+-;YNK=Ueb!9 zi~}6VXN9&0q7q+b0KWi_I6}$}@~k6wK9 z?s8NozIk_P`k+3F`9|}hiGD`8pXQGgR2w<<%Z`lN9OZ8?@YdUUx*_96N|#=$uRcsa zhFrPpnBVc-n_k}D#p-UQ3L&Pf6O?kjq>7P}Rh?D52-G!&C9f73MBv*>a5Mo_7eZb# zbi1d|O~!R_1;TReCH4mV45{X>1ly?Ml(hZgM^e`;eA zc{`1(K%4i-h2}*-Tg|aA*ea4m0H+z(Kl6m?9x_Duns_!=WRt=hxEbK#TTC0b-NXqi zmh-qwI(6%p*KLdG3sx-2m)BOpcKn4tI|4LYqJ8yPl8n!p2e&Z_Jxt`h5@kGU*s*y+ z&F(uh7vzoeEhUM_GL{6b0%DE>VM+F=ZaNl8PVfE=lsB!Wo#O(=)n?GYAcB_LuPkgn#URBN+ruO(I`FbLtpsMgBa;Sk1=Z7c9T7XmaQEiag2(zk2y#2Zyg&Hke?l zGt-(tG2B4E#yH7 zpeXGpL2IIa4srB0U(M6>pUXb4IacgNnyno z%Sxw?tJRzFq{0kAWSt@auRv_dj;?p5#X#ur8Oit^BXjxJQq)C*M2z9EUYZ(~4ja*| zMaim<7>5+JPqy|F+})7~b)IyCv0G4H`rv4Zj^%*R56mEGAFK&W$qUP7OptO>3at9+ zcO>VgVxCr3q!zkO%Ddw#Fn|O~2nhjN>qm64sLhHh8JB8x0y5f`EpFnn)ykK+mg;|T## zybf?M<2B}ZzMOj@LY;AKGdaECU4hE*I!FqPpg8744u?78wdZpguPsk!ykgI2yy8x3 zyh0v6yxO&RwVMySuURT`#d-FDd#n(YkHgOYM@7$>Y;CckJEN5jE3RO^h8g z^MFZuAP-BX{o;<~q)rds3trG%AnF+*2yvc#B4muE?;!O@63OE%36hhO7dGL}1Q1>3 z6)Wd7VW+NzffY-(Vxo1XsC~BH>;j z26Z{phAI|FN3Bwh)(sh(8YrL+p4a&vHP*RE*;PHuxLjV3Hv0EHjKNn{We>qB66YXjyM^QKRe5nI+(t;YFpZ0DS`W`?@Uxxkjn(z{Vze zfD>n%MOdiq#@%F$jZgsc0DkE~Xn_Wcq*jU;@?$k;O}s+5bZ5lUjYyDyc5W-yrs5%g zpk;b-HJCrv7Rj3XU7^%`P1N^v2EfXg!{BjvZFw70%kqDK-hs=C|HN#%oZZe zO*RxrmIY0i7k8_7g1~?N&;y~84+ZE)$T**(7e)~#7$RX z#2~o#Z5U?+htT5wv_a-oTTOcsWAjv6b z$_m7)#uVi`PgkbBqui3%zVH$l!SD@E(U)5aMU5zA5s!3-El6UtOk^lXlJOOOk0Au(hbL5aPHp;aQg7lY3gqaF5A&F)QWwT#BgWqE5v1*Aa`AAcX!1Cf9x@ zy78SUxV9Ef2U`_6bEqg``_dqr&EDw9yifhmIOZy7Rk1REPX6oU{qKLCoc`tH0X>2_L|H1*l*ctQ>+L!Wjmn_B=OL@H-0i*%< zyCd=dAhjC&*2GlcZGI=NhA*ed9coE-IHw>tIeE56N7@g*Vl-q6L{l^bLWq>9@iv68 zO)c{;x=?Hh|W|~NqN`RiJ z;bi7eH!F*P%OhwTQ7P-jvJzlDM$Jhow7}S0MN704p+xmM*Cb+gE*_eVLqtvLEvIb>C_W=Q*QzsjEaYmSTR4Qys9Zb9qK) zrzS*jm@w4LELTpg2CyQPObsyL*py;LWKIjY6cv~*Xh9QRc}f_TG`Ulcg3k_`cSe#^ z9p3CtTRX+4T9%?-49joSKAmJ#n$hj0tF>fF(sfS5sk<0gC%MqxA7}@m?4$yY=FY;*LWmiQG%W?FbV>NfY zuWC=zJe|-?ExW64UxXqB>`rZkV%;=VQB8d#C8&+sqXZWKTg_OSI+bR?l=6^cHD&e3 z)weIs2%oa4`O8bvkTJ(VB^DTCZ&N;C!9y1ud$X6O8Ye^d_CHj|)sugo{D)KR;Lo?} z#3|Jc5hX(O6K-6$Ez@>;CDQAHB~x9E=N1)X!t$yNAZ%R;2)g1aZWU;A$^K!YCv=fl z%>@%$u{;>CY_ef=Zvo%?9Sb4i;_)@I22m^eGFBD1{Fd_v8y*N zDMoe`X|VHH^f}jCvKz|{eHVGsh(a*20YIC#AOU4-9W{j30`1qyS;C~GNL3V34z2!1>nq>2v!K)5F}aBe8HMj z?iEd8jeKP_fRHuA%m$q+37c0iEi)yMSdZ8H++v!dm)XBp^$_xG@ZcdM{1huz(Uewn zRAlxWYAa{yh!+?zClV$uXVYo|qTrBL;tUwfGmH^nDOFSQRz!MQF(4a6WvHiy7-jZt z$She((i)`khM9(Zg3*>)CEbl0(VHgmKWUp8!&cyRfF-9%DXv+;8b}$V=s#F^*`V|A5L@D&Sq)y*Z=f}9{upZnS zk;eJ+I;Gp1hnwUJOzx3j-ohKA+zu99wi-Pq^@>(^3s;!vD3(MbME5--7|Zc--If-Xgtu@VckqxK)S3|c5A zP;muaUuV|*V^x*Bw^sV*W#ls|5lzy>C4kw>L?xPc#yJj~c3~-GpmC9}aWG#xiv|ro zQB)VITT120=`2$Ry0iFfd)wz;+?Wc7(mOWSmKyh~HvK57&DqFr5GM90{e4v3_Mc(Ay;JKX5v>JlC{df@{8FsFUA zlh|A{rr1G)m;?juzGHNV9$xJ|{1w#g(2h4-jJ;&)P0=Kwr&A~sNb|K0(yNxdOeX~` zt2Nqa&TQ+W3A<^Nkq?cI2?>=Faid*#Pto?m;(pZw#i;yN!(u?$|AWnzVW4E^?D%wXz)u725;SDv>VSBseSU zg8f0g?TqkY3$1%ml8bzzll86hM$6qMf*5TIG>?<)w9p2vb;(~`U1}zMl8YSK9^M-# zxS=JjSGhqlR{AjcmUc5*E3YwAs+DgxnkfdU#f}N|sC# zDWaA;Av~8A&9bIew_X4OBu?#1as|;i7Vc! z(MreX=t*d-3vn#fEF`eZt??9Hn#!}FC~GvqFGXBLJl-gVY zD?dJt+oH`h^Smo(x}830ev8|tT^aS%%D$4>8eq*J#1XA4v7!}EzyiqUj(2Tkt5ld+ zinuKbq6i2}db0GM?3Y#}*L`;&oz;a?M8-)8za!s)58a`J=G80{50X?5!dksh`iq;cx#mXltfLNSx=OrM zQ<;S7h+B$4?o93V2bN{(UmwQ~n7%!|=ZhS(`?)lxNu6ACl9r+GD$za@(k{^U^y#k1&&XR;K}ENZZx&E6}BD-cNw6Q*-Yt3PVG#)rU2$dwF1oM zjhrq3hB_tR=o0Yue6e}i1NMP9AqZ^)xX#-go^oXyOhJ+q$b(I%^vjolbAPfyrA@ z=|BaLtSK3110@?;8U^0@K-bfpRT_`V_ypdW_x`n{0T6Qc(rTHr@eEfmh*?{-x)OM# zHt`zp{IS##0_u0jI3C5s?8r{ZH8VAH+)lTHz@lj@j-9fK@=P{o=}rIn9Zht^SvY1` zo%8UY-nwNI;l^90n*35H_}LA}w>LyPtoq!pmvi!tCE}iyxBy;#C}=LZ$iJjKi}eVZ zrP4Tspj2HuK4GL6(lT^C-77r+8v9Gp)PbQz$?mzRcRiq|9rcFnUY0P#4cvG^YzTv1BJr za*((BvU`nmRuG$t|F}Q?dH2UV1M|??A$T&!Pp-`?+}t58kX-Y=F*3PW@v34b%-QY? zh)+o)@%7wG^8|39U|ci%U~6qU3IJ%@58RW5(Y=(YofEz)uoMF9?TOC8GmeANBalF( zCSe~E#!}t*Umv>}bvr@;#9>mj*0Xs~BlFQ2%gB#4E7wHHg(fQ`Hijn=x#X$NUD4jU z7)@ZTm;!FPS09vY3}{zd>DOH%ZE!*uis&V0htZW-F<7)E!^XPR)OGRpWxO6WB8-u2 zSPJ#L_jMwgBEjLaG=Njsuu*HRCwFWeZyY2#@LHD&He@m5pjF;%1j#U@Z~Gd%IPr=q zU*%pyrL1snpcYMBp&<7{h;=~gp)&WH4%?nYj zL@t-SFy(t;pX=2?zNHx-vx?jvqOyCLpOLqsQvbYy43d%SQe7_vlW#=@&wrFj!tU=d zMcTi|!LNo{UVyYQvK>uZY%Hand@jjlPDH6kcZ)5c8;Q*`?uEu1K9XWj`}Bm1vNeNy zYpV5u2U2}DFQfzZQ-~n3X(3R|k5+6E5op!ER;*kAI$RP}{;Ih~o>6#hxvs~B-G$e2sRnDw03MUz!6QhL&lHNU4B!+b`X+nR4;z9E0ca|bg(eJrl<{xkWTK>7kglY&Qs`km|JMD2f+tj5h6H0^lj_TNeET z5k=jElA_vXYu#W|F|qvk7~^iVO+9v{Dpjg-W56oW?g*sAhbvo~OEN6N?bJ)RXURA) z8TSG_u9>KFz3vUl*X|>#dT%fVZ1ePPXc7bC8aY?vq!9y*X5jcQ=d3Jo?q=(L&K32V zR4|-rc+(lqU4IPcnCv(mp-agX!)+@oTCp>%hd9qn*_7x*XeClU*Aax*jx+QO!6Y;u zgbTtPMb-@O{+y`zZ^Q7C=$Yt7Gxo<(Zofu7k(38INkbKu%+*uZ3()Ni09p%bGXg-y zTzpVo`4;%8n3FH#%1AmC`3jjPCJl&+@I1-t6yt}Gg(_YZ!{@gc(ygSC=20U9ntT@< zS`^FR`xq}IR@gP0#NjNA>V2f-9iwDVV0Jt=M7%3_%X|>TqFsjnhiQ-!stBxgE3R#gAO_n^j5Sh7!(+=cM6kqxWEP3vn%S=Du zrOR8=%~Ti5T#5&h3i7}p5J}0YkXayOD&{!7hornjtcp^sxQUWCYhq^r-mLbPL~74) zR_AJZn@TfZz91DX7pzirNM5gI%nNF|-x_11OLrdI+MTUlY+rbjkavb6WFUbIITvMO zLOe~WP%Ip$X8v1!Hr9D1y@z&OQBNHH@4kB*Uu^BTd1h!fA4Hj@4?GQnyh(DR2%Y+R z5@FgMRSDK_ynJ}z>4?FA+uMU+R9`g+M2!OBDa$Lhxl6yCb^_E+$t#YKd_Wt@v#e%3 zZ}1(P2?%+C)V?x?e#t5X4Fk{zQ7wt&t2(P_&O|M?5?i3VFKvUp|L({J^ZG$cP#%?$0+@;7;N!W*~ zq~4b&Vzp9&tfe<6H21TvDAi$K!a({f76!1yeIP@*<6$t5v`3-!b)1H2656rMe!DI| z>`azR4zRoVxbPcXc{@p=ZSJ!hom>0d%tu~8yV(0H+pyYSHE z-{%0>8&@_+-`lZh=r6U9!7(o2n!O^sK>t%2kEfUt4 zOkZ+&7gvUBwl!o!Hx3#Frg9aGadddwio|9L(tV`Au1?s#ew1y+K7V-+Q=H+vvpLhw6G+}nl zlwR8VUJc%tMcGTcHqnrlaTt4PLCXr%t@y=hA2F=Z`6AzSbzaU*LQmTd&ZWJ{uVh;c zIduxTEAUHOmxY|C?EYM)^ywKiAtqb5-A^jX(bJ_(MQIxX zL2XZt$cZSSMG4JG#tg8S<}hp23IaOY$6|YA@8<6!9VNOPYA6XUN+9EfLij` z7q6z|8?DOdV;Q-c>c3jkaL30S*r=_$gM}bQpZA{u9bJFZ=kQAmnBSfd)=Y3kZ7oyr z>K!Wv!wox<6yDsi|ME(GAtZ-VrnDjynMqP-)aN%s>+h7je0BBi)r*UpS1+HFzv?$| zS9qS18_}4HJqnLa`O^hOY}?L|2@{$(brhlU0L{r|&MRBx+AsShnaFkch@z^k)e8>c zJ}lQSQ9rld;vve&6&7{1O&>HgL}^o|1ZupJU~AACU?OxrHQ(cr6f_j)!rEzwD6v?c zE;T=$9#0`B<9K>(1hLEnOL~dwv#Du}TfeEj@TP=(ZIWC45*y!$*R)}lO$p1MH-vZ# z*sbcB+gKg8x#0I2%W-F0bkm&ZDRwreQ3Kz--n|{T8%MN4m-y0cNW^M|)o7L$@+=PE z(sGX7`Vi)!NP_|>S&(YnNzG!QO3Xyh;4U3t>RrrH!P;sOy&*E;+t+~6+1dEbQZ1tHr&lsS0 zo3nPDHU}cz6drFFu-}8(u ze1^a!&le7gmW$1$@#@-!{5R^a2IpdsFzk=FU*ACCUho(1tmvqQc$NeqlgR{3N+}me>KX= z3HvSs{q zL;$r{K_2pp3>~J>oBhbD$&wZqb+vrSC4ex-3n&{;Iyo3vDOh_W2v-B`U8oI>7k|6B zBD7YcSLrg1J4yykAFnaTd2|o;b363UP&h>I_8UIm@?!-o@3h*&kL9PSTQ=H{<@aBG zgi!n)Pa4?K|6P1nJm(dsl_*E~4O&lG6G|;BvBDh&W=tN$t#}K@yvz9}!6q*hbRZb> z%Uo$CA>ihA{PP*f*@JEqH_wy*&}K!b_L2Shd35>mrc|M+_^NG1iO6%7go@_IP05e( zY>)q#cPypY)&E)342SRSKFoaW;g4@!>oL>NW*XjFdr2(=(OxvyJ*!HZOEqt8f61|S zavqbDDk;4Fy}jqaWOeR}E7;B(oQ~Hv2gcW=Jv#wmRmWSj^!;{z-1c9=Z{#(X5SkOJ zM-Ml1smsEG~OVx%Fb7RO>?5I=Xpr1ym&Yw?91r zN$#^}e|qAb{EwS=GTM^#QN4>hi!Xgq1N8mvAP6vOwrKGAsf#PuG^pLhWPlw4b zbEp!h*%l!F5q18Wvp2F&UW0c)yWZ)c!%{Q!4ha7(bIT279f$>VDCc zwUfbjqWiHen=ksHh~hY0@1q(*$e<`S{LizkbOxINH0r}fGI2C?)k2(CFW~Nug^5fLBB+7mDZ74(C-Jaez(pR{Is^-_NQb3mBAjH z-rj75%CNKkflGb_&oK{ZLx!5ThT~8$S~+{^&|Ji{W@}DTihlr9XdDwUccbwhi&SNq zniY||Ww11`!f?1fmS|eh0?WXYR6OJlv`jt4qNCCra3O|ZG24j_%`*^O2R>c($Bb7R z`&lDIJG!>tRgyPONySDK%g}Fe>|!|)NkubL+LBcdjQOR>Cr_TB?eWRypMOrEKv>EW zzUmSoc-J4FKAn>PeeuQ@(ob}`T#1NP*dmw^m#{n||Ks}Ww+&9J{?$uZBhjlhVuy1| zUOM4-uC0$&%V32ifJ>gw9cWuF4QDWbBYd$m1<2JJ%Xk1u9?U6d7dDtT7-k9Yu_p<1 z;f!Sbj*+>9(&PSI5`&>!GO0vg~~s4GI|Bt5Y*fIqN9=VKqm$Q$WV88hW=)N!!hrMNrkIbG4snPH0_;3Bb^s zVCyM)pA*1UJ_Fq27$9>kL2b6jnSB>aIR@+YecnUQwq9*N?I@a=s#>v4&FPArDQ;^3 zj@xK~k}q;rqCFITHshQpw2CTbsGd$)S}VC)=ZG#D5oO8{9)C^}hoC-#BDWWu z1YEUn#;cN+YZPA0+$m;-^$8+J7+QtZ8X^12FYc4k1?^y+U5}_Q+2K$4D2RBQn@f%< zY6k^#@WAdaeuDlHDhY!#HaEww|yLiaqmt7IRpb3sYKkJp@*r z;i4`|Rv~WB5%wb&kdbi&w?>kN!Uc7Os0TxY**z=QnsyZ3@dXaIt-m>@0%tTy;wfL* zQ-gB^-O;6rwy1G-h)SH1wKzvnyknYC`Kob_r3eJa*5ll^BIg!0IS*8ob5C_ScPNgx zU2V>TRp&fVea=Hv=-j17=MEFLO`XoeRq8xctpWn+&Vy9!JV?#X9jbP2S+{eC zwZ2{L&TXo9j;-IhRRzzjYItrl23yqe+_jSDma}}YYMuwH=Xv0Yo`AK0zBXI$JkUcEZ6S%h0z%?O%~G)lGBDhE_kM~#i7tX4%!wU3*&k=tDu zxbg90s7~#&W@jW7`EjMqJB3IQ{HZSDReUFeUmd2KlW`(X)$+}cW4SZEr#z#x%;@V!5vNIo{fRedjuZMU=t4Y>eb1Nao~3!6EQWi+;Nz4phq+WgpF7vtMo32mzRiIt+y z=rd-ig?z8g6$&SAuGTtx2b#ydb{|10CI_{v6e7Cpwg3YuI{#+S(}V1(X0kpNYn)@g zp60*yZP|JEKVR`esW(8k`}U|a5FO-iXfe{I zWS?{EI1oA$b!D<+@%M<~0tEC*81LJjjngAklXfaAgdVhGU(N`l$x?}66A*GD*OU@C zjb!;!b=LaIy64V_Q6tV`bpj@|-^IpQNgZSqOA%zu+=&1-HaNZ4%-7KWSUGO9LUqQa zYVI9d%gtWIo+qZSP3pMX9y-*c&YgFR8BeII1u6WJX>es~?0Qa;`jRKH9E)RNUQN-g z1UuPhv7ipkC0SJ$*j#`ZP`zbRVnJrIqU9Y+tz;EDOIK>}ta+j|haN=X(RB-8Gv1JU zJe?lnx!D8C8AW51ZK1V=-_Sy$MMTL$V2Kp~4qGkb9ddZlh&o#CwT=CGr0sdg?yUQ% zcheY4&Q^;0DzY|kZ?lgfMx|AB$VRPAZlwASzm-# zCyHwnT~EYUOfhhC#fE@wbSo+(tb4`lVR*HpLmk}FvZa}g@(4Rq0@`TdFwZG@qYu21 zi-#&1Schp>Kh`~fl6;y%XO9;puDwWTF7 z*1Eo>888SU5!t(s3^jq*Sip%$vN5m}czrxmqj}VnJ+UxzElEDIgQ6sgM}zWX#BGGH zi#zT;wt!Fh7Pp5WZ^tFWii5^Gg>?QfFq#7@WI#UvcGSJv-2I(7{j^mTtugVf z#!DD@ne;h0;|R@jQDHF58JglpITDEtBX;w$ODFqml=d-3X_!%x$&#(; zXgNz=U#gk8(&A+k$xJF%t|Srj@xdQ%#@f;S)5vJc=Gr&y^2l(G-F?w`?PihbvBVhb%BWfLSz^wN*fUnS z@>ybxb!8mQJ*2j6(2PoT8$~i?iGjHqMKNrNajh$(NP8?X(fM+lG^6t8Mq8x(mKc&m zxs{vipUV>Ch&SfcY*}T-8#-KsH)EP=2)YQC|d=#j*q9)<1w>iYZqv> zt*+*ieCs%JrEgsD2!Iur;mG)F9f3 zBz0M`yvo)eSu(fd7au~>T+eEs#Ptw9iDb1TN)etN4A>vmL-77@_yF6yeND}7dto=VN?O#KbJJI}1X zG@7K+ubzhU8rxSvtBRHRbMjv&?|=XEvo5PH;+}Lj>d~ za%Q-9+t%M^)FbJf2e)Gi1s($-SVlpP(~0(*q4zuW_)PC zysUprX%&<5_{mw8e~DN%gm90f`WUG8D-Kk{nQoZ|ApUGjTO*>>F#ar(Xl#@kxVWR2 z{$rF{_|dyG8F-q+*O6@U9E#`WRQhSV2+e4HZDs4TnYLT7O^|_ir zwf|0WYEmt*gk2e?1!Yzbg}*d%w@5H?z;T&eL}tTFSb zTih{ES!#cXTwUnZ=BY&qK06IZ5PHhlonjqo;d`cVjl`#!eYN> ziOAD!3|AX88j%UyVbLa*v@UI``aR8f%B!`F2*d5AbdMl1dvlqEDVsSbv?xkJlO>Z* z%g;v4wuZ2`;I>)4;5jML4nU!ONmBqC7>{?^;QUu zQ|W3e`sf%T=dnZFf$TU^j~aI z!Ou%^Pi2Ekw&1cV$BYO*&|aJxDoK=NRc941GCMlnPP{Y3^eO3jN2p_Zr&w7AOS%@6 z`>yCC!>qaAIF}U7QumY3}u% zwyWsE?L<7}SgMVY1;1x`XH}92lhGG6xnp@szL92d(sfQ(y6|mLihHgWCrdlD1Cg?5 znnqk^1D%dy)vhB|%Mk@5)y*OPBhmC<^819v$fAnouAR8|mTubUhM7)hS7-oV?e|4w z&`hxvzdmLvlXMn}2D(3ZL1VLZ25<1P)8IK<&f+TB9GA1S&BNZ&R7mm#K$X2@Ahf$@ z?clg{OZaY<>g_Es%0AFlkuf>F`!}$)={PAgLe^5OQ#@bT{zoOJhxOCUag2|4=7lRg_}Ir8!n~Ik%l_aPk#hj989v zyEQt%-o&poQdyjZd(>|G?NU5UDlw@gn{aRJTU3Yd*ct)>qr0phXz6~H03$DauY`?v zU%dQAhU`X;Sbff_iapTyGh5$ph7Wp2)3jt#8V;&giMNXvSC{0kNNV>w z<)x@l{Cp`^lzRgex;1oL51LN72W(7dt}Gp*3C%_6K>XH9Z`JzbFIXCSLnQk3l!hKJ zGBpyDW0x~21{k@cCMjlm;n1;>3`DYAa&Q7Vu`IfT50_1T0E$(9%j|2h;HD6mT0S_i zlzZ`LALXE5y&NQE_9~~d%7Mx*B6RaGo(ImJ!UZ^(et_R7#QXU?E^w3t;5{$v`&0YL2lKjpQ5QbvUOk&&}BE+!GF4Hl7B9m*TC)9lC!5QxYS5sRwxnM*brQC`G-H6Sh7z>Aggd}i z`v9wo0;keRRk495PyY0WKTOF>UP6H(ZYyXvE(=;R5JoY}33SphThKzh2bHQPn=0&w z?4VHXnmx5;wp46(R9t{z(7Vu+fqv=DKsUMNf$@yWJ0F>gJ(z7C&Lj8Jb#u)3nms-u zXWY#O_tK2X6yMAN4^G!met01B+t2^nS>gMsO0sP&k_boJ$c*mDOzrrjj;O5cedS`0 z2)m8TyN=Gb9)U#Z&9NTHn~os6*TWdE102bCjX9Vx+c{=jLs*gR9Onp%W6t8(@^gDW zk@4Coi+Nn`@|YxmxCDUc?Bp?En%ianXg!R+D)vepnVDyKM_(Z%H!gtLYt zG7t8aOc0l8Ix@es$sF}dH{;76j4`H+6ma)qZURZS5l+cA`G|PWRzE-DfoEFXzQe|2e&QvW2Z_jB&s` zKui;}mW)h1+ckpcVaC@n0Y0=1Zw5r&u2-LxSbmIN$op^h3 z2rTOr%_nLHgB#B|RlOt7Jw&EN?^Qfn!SV+wZFSCntXT|58f{MHJ^D(3-WKD7(m`j} zC!&ZN9euEBJAxf6v%wwl!!_=NI0p*aSCGBgP-vr7p`r#Q+-0zt?m@2RUG-0=lg}7g z@v;=9_U5?V@Y00H6Qw7z5>=y8$0uB~N-q&uew83J$3&x$t-u6^LidfYc*d6Wo{PFf z%9S8T`llmxR<4B~-rHEFo0|VHnzCok>}rTGGN-~A<|a_*e}kNzE(4c;V2SZw7V511 z)|+D1-U99uZOQTe8h+YVZqz6Tn89h1Nq!00(JwcKEVTdBa-;SAXFOS7z8oJjfX|uJ zN7{G+^*#B5$^aXDLvy-7Bg4t{)faD0)k*>D(dEmo2ildk@C9#oPiyQ9!Kl-B&G{87P1SR9UUdO0FgUbLCi@+Su$AwCXx2U zxjj0T$_+#$cuNU=7lUp)wQsx0{O`GoG{5to=aCAcx*@44 zOhEynvr;@rScZaRf~MrE6f0IOSuL%v2-SRAD5;5tqeC!nA4OC+jMc%60f<0S)-(J| zf+hmPbRF_b7#l~*?pdbR{B$LL;#o$gqFkJ_eDcjTT0FjEvvWo9{BO9w&IjjlAIt57 z-V#a@M%V2apSf9T|7&7)mZCG!g!Z+q$OZUTS=!)+e%5Y z^(D(g9>i-kRnYIfO#T15zHV}Wwwn9@b$vZ?(ukVLr&+fB-0w1J?H27mOcAD7)!b*1 zViD#-z}`52W-I_x@>WzD(?tKmCD=iIB7n6X!1;Ld`g%&fQ)2qf>uZ!4O}tkVSAM3+ zd&n9?lTOe`ktn;@K49dnt-8USf3aI<;%j8o6vVmKQ=UKmxgqUD15 z-I0fl3cP8;0kJmQUZVly;O9SWm)}7E<>SEQM~rklhp z{P$)%8zm?r`twiEb0WwmyjZgGy5^NXpU&M3#!lO)6G(+fAOoC9$z=Dq0DY?&VVc#= zPp8iy?3}HNYJEojDy;v#O!K3pRVL4WTih)siO6L|^U70%I+#T=Le=!e^(Gl-dG$j=w~?%0Tx@%R$V#sH>$v z^02)fu0gUQ1qL5D;#yk}n?g>qPxch#g>3-Fig6~V`XmS z*oF{MrmUR0dJ4X%t)O@)52-Fu1FYV4`SO=nP43wGI&k&|&Rt{uSI69bt^UE|_CQDN z7RPKP;}0LNpYqXqfYh4Eaf>@F#~+k%42D-j%pn5Co!_Y6CCiKk05>UqAgUX8Q#YJEj$6M+`v!vuq1FI_etgb{UF^1?|^2LOf$&yzrsp^uQ(}GVDk>4vhTI6!NN`EU0 zmP}}#PO$Cm^JZAkphoU{#~^aFXP>pj3sKR`iDtLYlR~5tv!YP|vV}=kJXfQ$)`fIW zduAOC6eg93%cvm_#hdtIO;k(xfQqmoPU*OOSQV^yQQ*4UY~Y~y+T~f;7&q|@Gi00> zmJBf?We8%_W&t%(!e^c4mgHthb0Oc=87nR2K6VfiQ8OwrkQ+L1!z1(UHXkzyG*E6= zcUj$~W3Gld6&mv0IjW^;_Mo_GR*oV)@e~9N9#|_TrKhe_tJ8mLXC3QD4fHM;??9K8 zLEFDwzQ+;=sj_1^)Z<`J4DTr)yGS0W zuAxYkx?2|_N9~`3V@sWnd`@or-ZGT(0`Pb?+lD-Sc1C73zV>#CD`?tqi0}VC-5eV) z>3=`#bb(7EYH(EQ_(3Rus#tc_S-^v2TfD$@J+!=iwVlI2P9>u*RA8*HfHNrS01}p`?PRM1vaxkV`^`g3_a9aw!)uux5-a?Z^JR{{YkyP?_}OC zZLgz!Wn5OEg59xo&lC>4eUk*fa7DY7kPE8B*}zf$u3<|?IiUHP(4=Z$_!z14l$9A2 zn8)L??~v7212B@|>=vw4n~tSZq5)$vRxwN5LTuXG1_rZV;*Ap5FC{HsqvQGG3C5&R8bNidKn6Ed!lOl&AZY&gy(uvDBf^-|bc>2WNP%odP)~ z-LgY_oJu%im-OC1OGw35tnRN4rTulgpKDDv)sD{ZxS=~=mi_nktd{N_?V%4Vj+@P% zz`*=w@F|oJo`W`zUto5-`%OKwGQ1(I4>j_C|KD%^)*@8WhbdZxYRO8S`wa%7 z!fNdk1XB@Z?|1i4rk_8XJ~0*b`tF$|WqPg`7|$2eM3hV@)n@fWcjf(NGMW6Qaa}wo zwBYPR#c~LnwAK2a{^pM7>2va;mX%n&)9ef~aX|9?jnTpsgQIya&>5)tU4q8hjFpoG z%cn}Zo7Fr^Sqa3=tM8vopMExd0*{!kOq=et?!`g)jrJ)Kw_Cj(|BYhfsx^yJ)Wvhs ze-n~3t)>-Q0MtMJJs~|q4Zq;nT#57zQiU&-;QCePhu2*DTZ*hMY1V2to)t?`0`O<^ z&Xv}hCC?XiMoagh-w=f9dfwnRsl(Gmy($_LJwo?SkxH^;D;tO@3YK47U4Hx7b?fVn zOwzPpyMm-Md}#9nMP1B6K53+gej}2z+C27I)hsQVq)x(@?ix#VPf+XPvN1mN*%`zf zaJ5mkBj{w1rmzSYR7)@hi$%_Vva~JEMn)^P;V4sLPnjF|>NAV~UNVrj>fE7+*}y|4 zVzbK{yYQB_U@K-E_lP%k`PaxIuXJZ~} zra%oCz&vRv^ir&VCd<=8@Vs(-HXWj}o~?K#8%UK(OvwviJrE*+qTiHU=HvxkvFrsM z0KVPl1gMdg6E(1LQ{ZyV?#-J%4b8y@<_kT(CeB&Rt%QFiQmw9WW2ylKf}~s~;+~aV zC4RPcOvuw%hXUZEz3J`HMq7_=+qd7^Jx)_Jp3uw=jx7Vxqvu#m7Eoz)f7CZR@{zpH z$z|0*&N~1afBsK$|7`l%=hG+C%^Y{X`8yx1A<*h2JjAbV5v=XFbRmqv|)IY=w zArCS3!|CaMWU7$I6!9(hU4v(NX-b~3hANR3YeluT{Yv@yhEK`87#GuC%9a` z6eZEJRS!izcltrFfRF0g(;&$?K-pz{#q(GLX{cY4KG2QXgCQnVdg_dkk%RwmV+qrq# zLN#}U3}bp?)5VnB{)eUV=X3b{=iAes9qTcp-Z56ZEJabU^alAtXlY-uaw3zG7K~(k z#VfP%8oP{UFeF8~{k5J-2E&90r4zq*#|pr2U$K=a*HiL@tQgHD$pyYnv#nuka1SH< zLq8+)djx!TD`@&_)tqfcs~IV6I&2L9)i5bNJ=EeP?7uMkes+*vD`EOI%NNyh>;7l( zxYK{cel>&8({X25K8Zp2R`G^?j{-8BU&%07#&fNW-GL!6w>eRlD7rX^yFI6a>|^1s zr@@hH;3OR>tk5#Pmgmk@{2)JuZQG6k1nH zQSzVS0~n5OThH7!fJ3-TFWg^4u!2ru10xM1_^9fiQIt_;07hmN)j^6%uGAqBfYAr> zF;tS<8I?Ryjk|?t#9N`d@$9yT^Tmx^r}@zsxYt@@rzY&2ngC-RkLZ*BnhOy&-Hcy% zEa3!bCysci#ip`FB(fbWjcuzWw&=vF#{w3+jbCh9+F@etqoeKP6VhVR(L%{+2d(Sj zi?JaaJlnSp$@3pRzV8m^qkB}u4c6P`;3u`~B5U%o&3LUGskM+StBItUY3Q`JlN zeQx6ApDS97maKg5g;3l_*#mnbM4o%i3DQ0Dp!aa{H3R!8oO$h&dVl9$L8W{UrFVC< z+2yTyXw`Q3jd^5m z{KgP`<65qB*2XTf&`$nJAq<}@rowzKGwDybA%U`~x7 zX}q~NmO%^>%^6FLb%{GQ4jK!f_5*G>oCZnKJS8@JM;E_vg31g|wv}UL1-qJ4g zv@UhOhmA#@z$FvQQ0J0WXBypK^%yEDOFfIJx^0l=NIs$1Lnde{&(+AN|4lMJXURIr z*!geOpT&|&HpIsxrc7j+czDNBUMkiTx%xU!H33yEJOp&lnvl8SHHd+cNE2%ceD>i3 zDGl`<7UWXoqJ-#~xSlzy@z#0vmOw0_hDI_GC>~e`?SFpIcnfQih%DnNYqF+sZ9(Fq zy49bq7wqByy2DR5AT1!oFk%<}2^h*Zmn(hqCV6>`OnLj80 zb@JrrlaueCO#c1vzdv1{{>6TNa6YHb=cV&`<$RXz_ww{Fr~m4m3~xTV8nkyd;3#(4 zZ9KPO6K;Z-21l-}GxVv`qR7@3V;AB!B$=@}ve*TaUMoWY^sBv`bdz<>7+nK8UFUSg z6H>B=l0%$9uzocLvPo7sk+JfO(2SMpU*n)Qok3k-CA^r!>gBfous(YZnRa3W95T&U z=NNK+Zsr2Cgw9{T)id)0em(4Eh_l?#sCr7;xlytr3wd_i^L2sY$CWU-usW|_N7j{W z!9HMY2hTlH1zzuSa{EnDuri^NVc6yEYw=+J*4ejOs*K^BcJy?z&^Fn{?g1<*V>5O2iA? z@-lqOlFH>u_>DIc@;54%Z-RHh&e=>T@7@N|#+%@6KrNz^KrW)EKrdqGfnLP;XJ&7N z=tbUFo4psf@0-Z`R9BhWS`ka_Rm6R2Dq`r_E$+9P#~CsDqPc7lPB@ADB_5&|hUUjBmXCq>HYWnq zJ;!#n`$eNzIBBN5L7^brSj#rX4m%z!e)v9D1J6E?gcem@GR&*Q;82^EA6PrFRfAbU z!N58SA6-Aw6rBf~z>HN7j6rm2LMxUJf@w9^HEh7wUZrqce;k4%0bX^EomOVGsaBU7 z1+|EM;Iay%Z)+7-(d4D6Kn+nb&bW|T8aq@|i!n4aYtuja zLC9dT5jksO{V~fCt=NvRhFo=^u6_H$9l4@wsDEiZ8|32p#pNZ*n9>{NnNEsO+C|t^ z_rS7@P>HxSG7)8XPd57L_jK~p#pM6|@OS;;$>iUE`1|jMluL6(JMzM~%ETJ$22#n~ ztIhUao8+a2dzVbs8K`r58*>RdR^_py;>B}9UH)sb2!O-qsamOsT6n8@>1_xN-yV!- zuF7%d>LX`uT_F!`nwF-4n#J_vG1v(6d1$^d6f4qssN=S}W`?gX)YnQ8R}*a_$_88Q zjWioh6VX{BgJj(>wQ=ESO;Fu#k=T&yd^si}%{=0d5D|Vp+gp~%nHP z7AjUVARO)zrAIa+^)_TeWDt+x!`@pT!-sbpzsK<5!FL@fUBTpbU(u5KZEFo|(^cd? zkDT_6AUn538%6{ORVvhPtYW2qL1VZA%Po8q1|EO&?zNYXMGZKB^2Mi8AK&5ww=JkU zV-bO$FqCi}(BhkJ=ol|=S0XH?k+T$I5QKJpQ^v=r9^c|4?_O`r^EdBat67^f4M!cA zXb31*=YVfCRy;?vljr1Vu&dq?2?w(plTF|T@5wx|BkHxvB zd;s^2JtB}1s8!N1gFv90cbsBVYc>C=mgtSq7HnYxFO|mVpl}TigFl+h6s1R>phn3$ z*K~WavD|)oW}pydWCqCFHgFd_Y3=n8cdJ1`Kx!H0!<8`h+KS#WD1Mc&6zfyovy$AR zZha-nYS^qA?pG;3tUa8kyQbjw9GbrR5KQK2ow7hIxHhV|B0R&z}DKACFGe zf>@{{rxf*KxoOEB9<7*wP0u)#BclV4Zp)~+K#c$MOtOg!bfau7x})($zRy9HH`z3V z)|FV%iYGM7hEEm1#B-XdF{9?}fyj#HDJ@&5fkssIY`z;PrN$RLw;`kCA7XZ_A-uSB zYG2)RikNsm{7|imADZ~RAO0a`KS0{%;CrCSzjScmW60%W$mL_mVQ)Fqm2RM33mmpRe5{rzy1men+#5xuHi9bORj68T(k0?Cl*zt ztZWdFiAuo|D1#*R3C~J=+Ln6nfn6fU>Xzdms#41>&hu&_$_b8@Q@^u&RhdK+Xw;s) zd3{Yd<12i5~AMyfo0R7m5~?i(&>my;`G}8CrJFNoGXoy?$l9DK0pi4XY7kLrj9$GFN5; z7ctgmMd2?Mil1Q++GPLrS02T?NAd2FfBygMz593EwzV+YpYvB>DOp!is~O38wZ~t- zldITDsvAFaEhV{KNlt_zF*B$*01p6;J^6h8_q(tG@HHd=jz*5%5M8UWBytmvjs1R% z;vE{ryX`TGciyFrvfELl#I}!~d`jTtt}5q!t#C#dk+F#veXnoVb&m+N24E6N2c+G$A6l?3 z@tQzC-#2+i6RN7};k|I3$$+vlW?#$5Q}hoEz3j<+u8AO>qpj-Z{j5%+InHE6yYJ8c z*4gg6em?!TzkWRP-o`D+t%~V{LZo@z3Z)@1ZHwhM0drk!mQ#NHq-?=|S(Ba?#>(=z_>w)SkbHyGs|t>mGjF!-c9 zrpo7AStDhS`FcnBj^*#(9^QQMNrt;_^>!SgSx=%LFH@qnsgO4d)pH^S{FUH}Cp?Im z2e+-~o0yrqU+al7vfxUiTgSfJmL1rBehQgW~z;b-yM)CLtLaL03o;H zM?%mM+=3N1edRNPrLun(Bt34n1L=U;4s6jZxRFljCp#SwH6i+sPC^vdT>bF6aS%ul zOI6_>xZf5sV#`poX;ztNdfn_`3-N4^(Vx-n%bYOV+5LvRQD@V$>7#DSy#4T_>Dlz` zR{x=z6D*?jI|uml+b&V8FZ61HT2MQm>6(AcgE05NSIc?tk1)0bp^QppB_X+$~+uU@gf&slTU_O7io()WpF zH%roZ4*2Eyi>H0*0^Z2v##GNe(=pSows(nx&FrMTWGAM=&Itr%V}`g(YX!a}2qYWA zG5JJn{jyCl>3L?~ec`l1LTmDg@lY&X8BDhokmnLm+7!QcA2_#b(bF%txj{$o$mY;w z2}7jslIEzBq*Mb#dMcln2|ho7V(!2j{JUf)=ml5$KNoMPub~pMXZe&9`GPBW`reyk zrWU<5miTLwW`;h2*kq!+-7+sjxx3YwXh@fuOS?T#uJyXK;GLn5aW;X-Qol9Be7gbP zY^(l8edp?*@^5)#8fFUUW=Y)ysdRzrX-$+-0xPlzcobGdEC|Xqo%5~gU6xq6PY=CG zx$QeEyx#c%L-6I_45>B}+_7z5WIk=^`3mP6^Z$<~&wxb#gK|uToT76iDO+TH_+Nj+ zwic}e*Knys`V`h$(*_Y$Yt$JqlOCXazQrw1(Ca0a#9Bc~mTIsEcVu&L(zqc#5cR{C z?7*;|txH+*Dae{RfWI9~w-{^>>;T)1P^)|XIK)O|vs|Jy%-XYc4YNb7tSOOQGM_OU zJ-AxgeTgR}=lv+SquUTB5MljM?^|GLOKTC&QgMM7#Qx?_ zBXYXIt88!II6FH#`|B59nE!rh{(JW6=jOkU9)14h=YRdvqtCzm>lbHVJo-#O_vp(n zK7aHlbhft{ct#-=7JoWBBZ4Nm=Q*kHeD3-CnHlAOKalZp=`0cW^>jhivX~W;2%82v zP57!-iDcpkeCO}hXVbraIz2ljZ@|*k^~@v@>FLTex29|{O}HRj>aYA-!@Q3slgUSw z+vG9AIVEotVZel@cYgza&ic_uca)`%(UU?dzIsJWLhMsbN`Yg4Tg9|luBCj|tp6kH(?WJdoy+}56Fxl= z$mDyBx*sx-+=R98$3-cAOi;%L!(Ys{T8BGIZ_ZAK6-hkI@U)zg%-CyBF~G#O}k|*ghcFu1o{rIbr8lmp^=V-S~Bj zO~QOj?pSS-Ev95uA*l~GDLJ8*(Ah%EEOqU)TOX}q*+>1%T2ItG)v!j|#gf#n&A2a& z>~3^CK)4o@<#SNGfy+%H`G^SVI@>0kt%=Zjc)}Nq{#L@4t{9{#2GoKHscH<{vJWRV z;d+Av0jqn#YVe#!)!{H*P~~n{!dI(;QMEaRi$Z6G)-F#|vL@N7q>Bj_$&xCPs6voa zoYM(Rotcc)Rr;~8_fZ~pNCsOXGN7#I1T^|uqeo3ItTx)Y>0e!3-yrvCIE?o08$U#) zg;z(QH;l45xI?L|+!|%gxGD~yZ4@hJE2?DKzZlBLRqZ;8JjGfV(^}~hydv2XEXji( zfZl02(Hk3g1a)fJet1hZcI2z)yo`U@2IgvvPl?o`AG$Ob(@Kr{k5#3us(W^8dmHxU zrhMOTWoooS2}tAT^=FZ+&#YcFaG-fI5hT^;f%{a;R@jN&k*)?QR-{B?jWXGTq|7nN9 zNy337euGoLxhZH%8X7PHe-7XMgQqpAqIpXJgt3B@^@tKI$%Hbg1!6_lr0+8P%+SvV zUt>v=pAYnVnjPVqtFv+})@sjvO|T%Muk>7mhpBV>$|h7HyU5q>K~u&N%v2@+<@L?6 zl~@6C$6Q;)F0$&Pk)tVFwCgLnNo-#zKgGlSY(L?(ks4l|a zO%z7TKT>`-M#|qFLn(hDwL;d1ZLNFn9^S*fcc8|5UmxSWAMgFUxc5E@2;{08en(he zLPA#>g5qe!3nTmpf>R5}^^T~=Re1u4Z>r142uY=_v1Bz%=pkCEm$(d$n-LQwk|ab* zI_va5&P+ISEga1iN*@}|(@SRVL1GZ@L8>_N61Xz41pfUXDSIR*fqn6lk=;p%mr(eL zL_dM(CJuY^3A-v{f$M&0A49k-Rcp=JSlGK5U4#>!77#O|Rlcfd z!(Th&h<6WAhpESeNpx*iwNW-tp)>NKV}aIZ1kTB+sfum50A=+xI~SI1$qgKkk=LYy zpWSl@H2#{H`d#-TgSTsY0eD{DkS@!J-J%^gqzmGMZP5=nEB3ud#*S8QQkP6aK9o%| zc2n{8QSo+A@%By$bxjHN zO9^&L3HC?{cSi~LMG1FA3HL$?cR>mFKM8j}3HLk+cRLC9ISF?-3HLS$nuY@XOm^&K z;vx5_%p-{GVG_}$B&I(}xOlO*C&@!}BZ=)p647Qa*n4Eht|RssdY3S;-^f6JkpE?k~IuNI`jL%M2Sn7Y<@bl?rm2DfZJ^RgZp_ZQpHqEv^ zcQsqSfAuWFCeX3Kux%+2TWI@qN4b~X!+!TN7gbfq9&ByZ#Il=Vz#kq;boq2=A9F2mEZ~%;JQx*)yPcCaH*{K(e32M1C+^%+_$(AS%YhyM%3vWQc*CjK32Fo zmcKSuRAsDP>SBc|nLb1f(}%pe_ph9{L*2ZQ5PS%PV5drXe*`KJ0BZyVUpdZB!Nt6= zhuF9+dBt4IF)l7<3GWdR8ROy_GyeeM;!qjHe}x6^GtoR;sg1+i=J#(a^O7;$K_|*w z|BWajdXuWaQA$|qk}*vbp4I|=aZe_3KPU7MxY zGi9#{E!;fNYg2;ne=lO^ugoz?fRp6UIc6J?D|@MNFOUsKlZm3k{023P9~ zD02#T@1G1Y==ihSp&r9UCnlM2uTVH?BO}vE*yhzP^G_g{)P|VpqbP5Usaj-qWr}P# z6iUi8SR2Gf=N&LJGFPal6jLa{rYork)m`MLeU__s^axfs}eb373`dh9ysf%r8-gWv_)sh+_J2zPucWCDboU~tV<_Wkq z>K{y+KzvEQo9fAjtH-xTcBG>64V)NWt*BWvi3=Bui?eJ~mSFKO&H93eit4s1Q-p|P zpKLHGS31TTi^^GOY1Cu%t0hV_LUciPyfwREJUh1#2$D(e>@Iz=1g*CezSH81APGq| z)7rJLN|}Lo-CJY3bF;qDSKfqNlptiyxY*jb`?^yw8{$s++8C$oFG00*RxzAqwFw8z z=USmK>mlG~hoVT>hy}KQw;-AVB(4_JD#q0kZbt*LNwtb-QLBDt3yN1smGG~&#n%fk zqC)S0HbLPky-{9)>NeTrfmrLYC*`0j>1{}pXC0MSuuM=;2xyWe$fi;X7j!|j+HC2F z51Z2zE{DGQE?Nh)OskfvRsL%gdh)9U_^W-bzg>(~SV2NgFoQB+wh2%I^$O{4q*B~c z6G>8^y9cu6#sA{7!$X_$6FL2V@!79XXdpcAcaRm)Z;}t<{*E~&{wvbJ!`NICFUZb3 zl;1Ju#EsatbN(EZ>M){mQL4uuAScC*{pY5*v15)(5bk!mry@E&v~hy3*^334+e5zS`eca9LkAl!HhbN zP>xIuW*`%$1~dA}Q0`3iWGIKGdNPztQ#~2Vsi~d}<DM zb9u2n5~CIfet|)~5~IM{#(zV4CYCcAvWejyif#<KO&i5dPJ6ZcG@DPVJY0cMY)*AeXN z=*Ezs^X3#Hc7DfxkAcs-ZjgifK8BW6?+O{)|1kiwqdR0|AIJb;f0xKu(Pe;$`L462 z>OlrE0rU0e8-Wux(!&Xe)~VIAka|=_MEr#HhnAreJy2ahMKE;YVH{)Tqt-9BbGla+ zpe92pNfZ^?HW$c79aEfTW$c&pPP!N{-?u}lwI5O}fb0zh7@gRo6MJ-G*T(3??tgA{ zV%HuVo!Fxj+cQQd_UObOo!FxjyMK&M?9qumIFMA&OfS5_E|!kD}Q#~G*h}d1|a8EtPq>S77pEsxX@+RHn#iLNugO|ihtZe%4B>X=1R zuCZqL`%0b0n&FL^e-JgpjcK&I%R6fC-}kn>0_isVGdU+F7Sfb-~8#JMzX|0$|NYYZ_w3Wp;lH0gIy?LHi${NY^!b;6{o?G4N1y5EK0W*Lv$H>;vpucRI|`|= z_|w@L5j4p?&q;;nbI;$;%qai+fsBt!3M}yJscE-XNFuCHyVZGZ;zIa4f44rH{`J%8 znQOq-^~@v@>FJ6ys<^&p(}WAcrT)sVotXF0WHR}vqRo#rohEM-VUUh6z55%mi}jr}(bn2Jx8Ng9BJ`BN9r+k_KV<6!+Nonj7F?9_ z#sqb2I{d|KQ00|BU$}gbj7s%g*I%DeEBWSGA#k?63`3}vT&P-oJTnDcf|ry*kiypM zAKCr~kI@TPB~NoL*ECN1qTDN{r|;aRa_<+rvZCi}V;I%**Z)pQX0m*ngUtL0-f z)f+ywZ`i+@1N))zyDiSp@!hI4Xi6kC!5I)_CVM{H`m_1&Q}b@)yY-MT?}r7!>Bje2 zg;gP^rHA`f*9=~1AfE4U!%H|bXC#06>gfNT>X*>}{@>{E710tL%}9E9{EL0K^(ps2 z|31fm?%XY_=WFHPqdw*c#xxNGXvGa(5vlMhZvtI3!KJQjcI%*itO<3(k|!kW`YA3$ z&M9}p{v;e^HgIVSM^`zf`YrbswloK^O@Jyk8qYRv3U#%@Rz=Ao)dO{b@{_* z*NtDdJ0>(KMy<+h`|_cb|L)XJK5VzQoA0beV9B> zW9Nq5ZV7mN4r$^bk4@1v$emJ?wjxV4e^?Wt;Jg-${#L@4RmO@UTUG)fR}9At z=>(?EOxVjR{n(+NJnWERwMfwMn60$dWV_u!raO94HX|u=eOJ>dx@71HUXknxmfh)R z`#b=>({iFWHtq=Oa!dQ+EoH=s4}Is=Yyo4na>uP^WQo_53-|lpoD#UreH-PP^;UXb zZO`gWdUxDEy~GJ->BNx2<{tJNUn(i$Ty&+WwUT)nPrFPkNfcmo1q)%ZC``8LyfeL7 z@8e-5^&C1A6A`s|IZY;5YFjsv>0%0TYHQpRh`OX{N*J1Fcp+VIvmmC1d%f9$IolMh zX=yVgsspOARYkpe*Lr$xasw7cvSF1E; zB}q0v*7JV7>})`l3%;f)U=1sm={|B|rGbVJ00racm}WiEjhG=Kg#MzfET8ekf}}gQ zqd0df##;Ii%3c4~9msl0T!ieMK3id99(6-uwLx$l4s5RxSDJn^5hUX{odAPfus5Emi1fltNRD&iY60l5Sl8P&Zdk{ zGM?PcX-1BBTba+uH-QOu5wzXf(N`qXG(YK07j8*--LI7dxoS{f^#kpg^tO4wM#5_} z#5L@xlS(a#z^^;tHXp1yXqYf!XcI`vMuDw|7ldgd(rKo(WTrN=m(JRN{vST;|1Q{u zmw4URqaX58as3TjGlI|OG@;GiNO=MXi2HJ9_n4gzJM6OaJ3e*ZIvm;9Vnas@=^D_< zoHI$muf)*`-*s(=EKIgImBu$RMK3tCBEzao+tA`B!4My=*%x_}b7@Pqb1CVJW>j@I zIYcu-@Et1hg}^D1O~qC47}* z1qY>k8XkSzXmGp`&DipYr#5x$fcN>b%^@kmv~SR)8M%bE@~zqqRqG zR$yQJUaZ|cSYAJs-$&)^Sl8v7@^Y;qJ}Ep0gK`IIH{IGbbHo)-@g#Wm%o=pbjrU-@jMNsT2ywJ903tpZ4;>6%yWxd)`p_SB!& zgb03E2m-7d$zr8H_nOefQppoUr(|k$Rh%pX+i9kwo7#VuM*smJT8yZ19C|hOLxw$Yikp8?vXRr zRFIfeflA1AOi)CuB6a{7M6x}=V10CzXEdS8;wU;pM}VYM9ZI-l6P_RYgY(W2D@wZI znV6Qe+_&K)UZ}2}Sj-$^i`i7YF$Wf01q>_et?EUcAS@+MC|HOhFvW}}`k5`?XI<1` zg>gHV>QWz@ORkfe2$-OVX$XhX<$FiGHK2T>_k7y*aHAli+WCg7;zojO)Xu#}ZCo>k zV1gNub#l$v&Vs#XnOrl570WeaV0BJ3uyC#!ttub)ubOM;9#=be0K}@wt}X9+(k!2g zzEvkt3EkCr7FBfDE@=2+#24T-qs{7iitnOJfBm-ByyHsn4ngHcRownUyaPbiOZEoJ z@eY96EZo~wC-_?xTO-(vXT9}-mv{|=A5>Kx0MQDAC)Qu!Z1P~3&IM86Gl?g)$a;+z zL`*|fhzEdd)hmv#MQmpj&(h4;;j#^>PHaYOjbbxmDiymY2i<_^s>Suk<5mfR&^KVw zc^_mTYg-y+piu@IWuQ?8>KvmCG|E7u41`7*DExTLDhB}GC9T;EW5F6^O#9K;8Qaz~ zSKoQtN*ml2E%>;;Q#W9mv#xo8*@B=)lSgM~A#AM`8~yClsGSWL8;!xRNzEqdea!5% z&$_*zQfYmjfqQwGEbT3w3k}Cb-5t~w;94KnD_~>xRzxAJ8s=uYA}2P?AT{LFgsggC zIxkm>IJ3S5jZxFcS$iPc_1IY_5Q3n%Ch%_Ne5GefnQ~-VPn)>i^?){kO=&@F-w_j$ zdu-M&H&T(6sRJqCKYA0C*!X>TWKEbVaF%TlSyS7&O-~2{hp95Y&-+##Xy4W$&D&mC z6T-6%3=x2g3 zZ+Ol#zSz{y0cZobgIp6BB#W7q;bVLoFIopX`Ze4bb$b68b$ZVjb^1}KZ_Rtu=|`P@ z)ag6Mzq?L9`ZY!ws2rmVL2C`$6fkqieQbtH zbE<=!R(0=MgTS}*U4s0)Jk5vBcL6QdG1y%UBA^BTcR|Dk@ih^Zc2`z=ifNOF3=gBy z6i?9?1q5?QqGCmsotVE4K_oZaRf9cc9Fk&VWYnPsgUC8ABkR- z6*QX0uR$PD!&J7D0SxV`3hrVEL+x!Sk6mn8#RhIf-GZ&MAihnDaK@h6=c2*t8BvKeKCN*^n6j~t zRwLH4ZOfnW)Z+D$OJW8TYU6c~JN;Z>nEJxDf4}}D z@SHed3UtrgXLV)!7Ga5QL8fNt2!J2oR@)WlBUsw*;reih=I+*lBs+FH4r0%Vlub)k9^tyM!cDNDBEHJPRD#IXu$7JKaTzti$te_lBt3rZfUI-|m4bVk7#ol&DRYIH`8&L}rVXVmD78l6$iF*>96_xSys zQI(oASV^vHoHkn9w0p#gCh^)_U73W>HU8Mb-rj8s`1DLQh-R9~QSHEc)11^}k$ zq=Zt13FkbGL$XGdutxB2nMG12MpW1lM|fW^iD;~sO4nxxVTnY+p#K?XHg6&lsHWt` z%4_eH(f45Op!8Me6~6_J2tb=gTV9AW*V2;Jx^e6crQ&k!5WY5n@x=Dvyu2^8Gs zz2Ph~rRzKpol(Ig5q9=Qb7NS>)YGy-edVE-=twi#Y-4@bj{12?&o=iq=DRgR1O(WmQbbVxRKhL6 z!1HXZDSc4h)UEh{F1X5&~AgZ*x9X+*K2Bsa# zLYQ2$2-gzR$%xR2t>K9wSpPV+6fttp!4lB18`V4fAkN%k5OueAP#$~DrDNxH9y~UWnCN`wT-PqnqP-7yA5Lgn`0(p z{+o0@sd4^q#sMAsZ?b8DjQckqp)&89n=k*rJ-844<(1rw2 zUBx&FaO1#>m_vZ)vg0(C&u$=}T}LKo7|rD{4(!?$jRpNZ3grIJWJGI^#``hae++0IKD72N ztRJtTU-vJD_hEM*Ebgcs9KH^F_qzC;|F!?TH_7tEh= zvtY{bZ_ot#{~JR`MNS^vt?bshFIF5V+;bAdv)ic}M!?MljOc8Dht!D4;m-dBl2h@7pXOUOVu zharP_mU4iNLJ_-8Xw+~;@*ZJrB!EKm2$z>slMo#?7D;J1D^npW39Wz6U3NaDo~dz2 z{lRqTsT0bo$5H3v;YmFm^z7*0Fax9g5+0|Ei{Cr_e_nSgnhDAOL1Bz1qU|*g;>@Ud6$?3^b z+OGxA%uPAnP?ATYCRvBl%m6w%W!#xkXkDB?fmBr#HjM=$BU0vB=EH`!Fwcs;bc7Lpcg0XEcFmFP&yF@s8Z>5 zecvUy`AUZkb$b@+LhHO^(Rl@72kAO^8}r(HvQ!{J03 zilh+9%JWPB>bJLj>ltJ}J_=E0l4zbYA!r|fWpQ;sv-1i^dH&V-0N+oa`5D$iH?jae z(!WuL75ZJ_tkUUAq6<)f>*&h1ckgtgo50Tg*KA`-y{CiDmRTwvt33`|^)StxbtlDs zli-!9plc&W9}`#O#zma7vmD*d@5W> zi3d~_}}-Bbbs+K%ygw z?3Yn=GvX#fi)|KCuS)%Y2SPJB&_S$w-q7f#R}xV7vcbW`oPvhHKqVs5NdBrv?>|T# z$7l+KMJ>04UQ$7e@iU-y86D$0RGA#VCo4_0*amJl_3F%l+gZYEw=4TP>s;cb>A0?# zU0VOVovZ@h1d~%1EM_5`AnV6`Ee-?osZ&>P@21kvbvEV^Z;GW#CL`QX8qhw17Qd$>h>6gYt z^Jpb|%7x^$03RR+GSOfW8gk>36A@xGQVQJ^Q;O1vODZucAw{8OEI6O9a2Qu{IUlPH z#23@WAS!8Hsi8x9_(q8QxKguV|ALA&Ior#aI54 z?Qph61WDEN1Z^&0p#i#hvN0l^i9ssG3IvTJ##XTpnMtDHjB%X5z}9~Db~0V*UQKi@nS<~fU@qLZ6PmA~mD966M_!@S_~tS^_PEQM=5%a?>e=#bTHt zm_MjrMs#Sw$8(SD!}tLmBTqtWKKM?@f7eZo9M~FXgvQal&RWrrcJ+-E!zajq%*tva z6Rn`=B7@4EYxfW?`H6Jgd_r(7w>w7Y2}|kI<>!S)Dycz{yDR%9g}8A3i1LWm`^DE; zn)emD4TjyXUF~H;uFj6kcWw3VuKLksBdrekeHG6-PnhxS=q&AxGj|r^GD#k3`Re+w zDj7sN5s|(4a0r*ehm!zcNWN>EW4uVG+R|{BRmYK957m&oyS%@lFEJ;TOX<0Do;X*d zAYL0~cbWZ8I@>}zf zL0*O*xQ@m=rQ(F8D6zbB`a?79o~><8v0Fe*#Puq^cT!?cc0GSm= z@7w(=Kgw+zQjMHW*^*0--Pdv%ZrDR4}0u2rB>p`em%5$I8D%&nOW zIxBeSb-hDU%Sdw|Y?(dU$;@7kdL@(GCgQMl10WKrw}1(w=|e49Sdl^W3Tp_Gv5TBp zok~rhxI{MjT9eniF|QB#jy^ zR2jED6ps`#m<+myfKpOgw&>NX zg31Y!%&n5}R$T4;KtxkdLmP`X^{Z8q_{=O+xkTkX3)DX*$m@Xw7l|5777ixK7GbD@ht86LK4& z+~^$Bz6-mLzz$J*_9t6knndFQ-}^J)pI_}B0_gO%yWh`Nrl!!@1+C_xxdcd!$_4q! zo|PgQSkFN>#s72lo|GdgtmgT*@Td^wVZ8+Hl_L$8^Zl8vRE!W<&qH6wijYETiYSJe ztMpfEr8@q!c6(#i94~lpuk+>ZxaBjBLAQPbvO{|`&O&w>ko+Pf&_4x{;TEzz~z3wKPe^}Ot0G>L?8CvR}g6Te3ch1#7?`&hUCnO*d?cSxFi?X6LDBRhC;VDD9+MtYES?d8kn|dym)q_72dd@e+T$0A(ypGW-|3G}4^&Wg zH=dXVH5Lqlbri#J>WjUaVtAy|je~b{_AEd;2N80LW(tvd(3Q!qGRO^!i4&1mc0bFW z=OTj}ZDuf}l8&rvIBft&2e&V#bi*orSIJp|QE~=i{GFna85jtZnjj{}8B=9|_0Ojf<82 zBLDCW*~$13pgMf^0_3m48?S_kLKd8qTPn}{FLGIv$oJyrE8>s!(#~NCu+rP$ zpaZX~#8sQ>iP4HE8(P#TIdXS5jLo3{y2?KX{h&=VQxVbuGO@<#5T;QyLGMP7Ie=eO zTp?^$Q^*fOmL-0(BT!KC@C3h(WoJptdtYW-Jcf$CD}$aD!^`2H=pC`+tZ z$q7uX2S!G%-;5%gqPp%)f?eBb>T&EU!wtt~@Bx=Ul>oEmKZYV@Job87+&0S1aMsFU zVb5pp=vF?zNVE!el4|BHcp@lFs{+=lPHu(HG{Y#EL68B|RTpe#%xru@ ztn-)=sL)uxL{x|>>J^e{Cjq>C(W{X&nVPAXM4jcIq$t}`aQL zNm0f*kJ*L@>Mh%AeG&l)&7_^x-!RI;@t5aIWTfsi@1E0KK$Q!j(8N9OZ7RU;b`qi+ z&-OBkomY#usbL@v4N2t;?aabbk;BkHgp?BsW4F+q%GA~!mEsX$2}u|XM-IQD(={lD zX}Je4fk&&hwbUKNJP}PTz|qSpOoJJp@qEFuKZf)YTouXA zpL&C?t&Le;lh3^xU6`yd-z$>US;?}i> zQt`tWI&H`%m`Ffsm+=bS6T0l7fH{Z>@^g88xdTB=!6{nH#-&=HrR-!3-JASn>k)_% z*%E(QG{c;!kYXDUN2o`Ulo*~eGbH%KnQBC2R8)Aw$pIw8%nYj+!PHG!A|1%k!ZPPL zvU$Kp4i-Ju+d%f-`>{ol~#$awveEk1X$K0 zI@PIuk<@?prg=4+Dq(`02&-bQiKg0hAQMjt%q}~Bu!{PgfaZO~%r(tos#8S@g?l=K z`{9GR2}+&(CEy61HdKZs?#1;b*<9{r%w|AWx<}R8q-0Yz&{>EtPqE5&bo>)^eEI`j z^zK#B{wb=Zf7&fgi@Ac1b^)Q(4Fzj%CWr+}cir98q5CBdnAFLqh zJDu1OP%XRUWEW+ZuZ02eMWWZ+RWY0&;2qRD^*Oy3{#(DwTJyuU(Mg$zX3pO386^U( zLRE7SW5$x=6e%SMB|s>e@u=abC%i7v-Ah-w#vyW{^)o+d;w;5X7l$^R1QV>6NK7*u z>|}g*U;6{6&y}WHcZpshree}o`aTlU@ikBkDIG5}{zW1C6tfpRC&rr!?NrmAhS|0a ze;SH+7JO0++zpS~C29%ij>;?%>9k6s_Xi4uXyYkfw9&oJN}9jeej?3=lADE8*^Pbjz0mv);~t z-XRxswox_|l8#2n)h-y_%r|;oall}da^gpb8Df?aSUN0z4F1|_aSYP`5t>YT(R+B` zmViTZcaE3)y}Xj~y<7}`<-7>XSEcBe)ste&=zq3$752YCCEtIIUcaiP{^zKUVf^2P zv3!-MufdYYMZYeAUlc*{9pmuMxiHciTtUIyGBxrPdcqYTYgd^n+yPZPK-12Qc==Q6 zDwvJO^Wu=~Pu(q@A6U*#|56eA&ve>X1NepzhV6$=zke`aj3w?#D1Qs5--Bn~5tsBz zJo@gi;19VAmwu#b^3hO=WZhVky%8*9B>67k;O{Qfcl6pu{X$IgDSh`~_9Z>y`#UACYw ziwjX|J;l5RV1u_b>xK`)B0gok(b5XA(y~BfuvSLifD{)bxkbvo3{nITxF#$RjLO#? z&WGi!rwx-9Z6~ws48umObjUrZ1joKU=MO07(M&x4o${=YD=8#}X!wu7D2v?7wIX5m zZJ&C-aRDVQ8j^}dGm}Yz1!Wq2LB$Kl9G{ml0ac5oxq>t|&})eN$-6_D70gD(c5&j` zjAs)kL4`rw2WGlP0fc#wz?F1n2$ibgD z+g&4deP#`Vl9MOom!tdNLx#P>xE9Vo$I3ns{)fB1k-#&Az&;Tl5BPiJ|!bt`rO`;ouR8@ke3=w@%( zm(R>UJqJwTpa1l@KVZ8YjBO|<9?ZC|U#w7(ww>%Ud08@Zc|?BcENnp6XwPBd(h7sB zM`oDXn9HMRs(R6r2{*f+tcUQx^Tzn!twqeC8lFcmx#Nwo8g@2vye2FcHv2s;WB@;X zWJ34%6Q1~0GoY=1O*5d4caLY5Vi@N}whi%AN~!S$y(O>z(u}g{6tjNpVc5GF5aG1S zGE%unht+hk%Y8)+_r>%EqmDq7n^IDTX*v>woANilC;PdNfTP&&<`5`I_q?g?hv?i4 zhi2s*{k0*f%gBcD81^Teev02IM{82dgAmAdPMFth6_n0SaXn(}guZw@ zeN=XlO=i@`Ld$h*4FENMWPBhaj||H)0cj$FpHqIUf~611I%{T$SNk%ubVJ%$9f=M| z84$2~d0$+)eF_g>K0TDGoz#N;*12PqjeTChC2VGT0t(goAf+|Pmc`ICK%k{bVBxUr zHQY9&3BCLv!E$MCM+{mmx_S!c|Uw#Ge6VU z=eZK^fX=4*!KPvk3MdDt(cEVe#6XGt?W+H-FG4CE`$Ew*W~cBJMBWsP)!&E3w(chl zoP5?sLc2|)oJO;Y;el_&0paKK^>l<4%;*TnHb+hZ@DHaKFtDdr*n+c_)#5%#PC;K0`}uADu+K%B)}b)4(KGMT zvSJDo9ykAZvQpy zNw0Z|L@qpC>Yjv07pwKITn()R(yugtVZ7h=!P(N-BnH_(tJr$!-Dx&EL^1@2R!2hC zK;x)x&W?Y@4(2b18&}5!xpkKY%ngS+orAqz@_YTgTnhsQ(ZjFE4fUmTQTWB#~1L~l5`L3x&jC|O1xuu+!>8X zb$(eil9od@9$F`2QO*bw(adnJEQ<2d^`KYs>-ok=;~y=KwLdY^{r^q>y~E7W_NQ;m zIL8ZPo%dCYe^HXKK3WZ%*S(Y79)L& z_0#je=&%2h{AarJ935B%NxWls9Ae6{DJC%0*+|n_>WgxAYhOYRjIZScIq9#IYQoRD z!VX@W_$2=Bw;Y`uwF9FM1BJv}J^GwdrO+$nNpaknQ@Fx$$Td$Mr|fhS4TBw=8l#wq z{uzVRbZmfxT$@0^lWF+gTxfqMyU$9iP1ktl*uu)xE8r5Y%HU9y_iJd&h*e?pmZ59) zG#&48MbB3dhyQA{qQm)pmM5OrH782) z_i!o*)l{<)Yw2cKn1y5;a#>JYbcM1^jXnsBk*VhS(bq94j1 z@sd=HiuvjZ9QcNz*hazF27yxzN+R`R=Y4U0-&0!?_v+&Rci`TK^TiYD|M~V>b=3Ad zyblK-ANT8N=h5E2md>wW%--K<E)c<>0X*7{^+QV?_TZ-Z?%!E%nz(g75Z`B z-6M1_pfp6i+h7u82v`_?8)od9Q02tkaE;={Xk$GKQJx=xr^PWwJKf6lU6*l1(Rm(c z(q19*HHxZ++fuP`RN_aNJM~A660O+7z%-L9O;?&4b;N$oL5ycBFst?fCX;9gH=})C zLgBvj1ge_=nOH4}9g?sj=lH{0)tCmukhD2grbWHFm^OzDF0%q!UbLHkAh9vI)>3P~ zI=A{9>Auv}(k+yopj!*D1-*EbZPJ4G<9;|}7#2WUi<)?K;NNz>n0)UVo30;d3n-@D zLF}n>c}FGh19hQx#%lC(bu%P7Y;X^HxuzK^dcF)^%=1Jg_H^UG3b;F~a?hrA4`YfD zRq3ze?DOd5ulToYlFJ5tpPL}l?7b+aCAOYR-a$;3c=%@AG0iX#O%%*pYI?UYRei>% zL`%o^#{<1}fZ-MadHMZkHm-oOV#ka@?WP(I*Z2aBGSnZKR}Q!@7(Lom3J z$xpw#M^$XKv0^{AvIAp}?lcoRuN^Q|63m_>J=xbCx5{&v181^?y_ zh&UrdpE(QOz69YQjz=uBM}R;|q;Mt?P!(jg^X*1&bsoU9Kk`R+iDe#Cfn7x9@Ve~( zz11pV=&A1L!ZhKj#dxRO#(GEnAht850(N~RHP1RT9;7o4Voo49+!ZzJP0(Kfkt;c~ z+&q7bzMR92b}a=hlR>GQ$lbsA>eKR>YFr@S|7QPl-10udF=r!1p5K*|Ci`sqsV=7X zx3Z69foCY%p}vFFx@9X{mF}ch7V-_b1tN|C;cc=&ukZEOLXmO5EsY}No_+G?6GrW4 z(qol=-CxhItGA`IH2Qhy+6e0wO(|?i%f06b5wGO9K*}`bdTqfNKIu2>3s+MNsY3V= z4=PL&jn(OLpZh{=mZ}UiWol;->x79(d)f}n}@=@g{P;^B>4O#c9q1S^5 zzPOZ8R8lj$5g{@H6M5$br+&MY{Ak(gKhBbi(#H>BBNeWYVyjY{))PcO`!}5^%u2FFIAH zih*kzdWtJOLRaml*a;{+?4Z~St<8CU3S2YZAMT4Ud)0)Kk3fk38A{F1!!M)PZ1o)J zcrUFkLyYC*$BJw|2?J$>1ipq`cY7*%u-8|eESeUW*Ix18+1&sOtIeWV@Ay=^8=MMk z1*x{D&$HrLK%$@!`FUWZGP^;Qfca2^CHUs;E3D;S7?mVEcMoNRxW;Jk4!Kv|+@opGb`=l4Se`Y|TgqB9TJc97q76!wRV_F5MA-9TlKgC+uc*j+Zk zXaY^5X-{A`Gt6dlr9!>x)vo9+E&X_Fi+TSJlr{COgBi*7U zj=d6oa5Z(ILsZJ&QVbw%ws(AoZ9&!M@agxsEof#LGj9n?0a*BijlH~NsfU=x!&pf< zOf#^K-T-yQLpKPyRguEd*-S*t80U@tlq5Vyv~y@SLo*VAGEg&0eOp;2=)qz|67Nj{ z5yod-6pCz@?VV2I=c%n2KrV#qkSEC{fI>-clVT-d2W{}E=R z2gU6|mWV%AvUmIOtd^_|9b^iHdu6PsfV*8SWHZ%S)IDG8gx2<36Cl(Ip<`y?_nn$m*hbU zxp;VDLaqT*L*uGi;v#3*?$) zqo00@nO~6mN2F?h<}Ap-+WSrJiV{qRMbTz5aY9!m1%IYWE>7;LMKPJyL|1nJ(BEgr zYOkb@qnd`h$!6*akWFFk=;aF8mrHn_wmZ7?V;0k>RJ6jkerQDMkX(pfSTS4o?EB%^ zqN@A%6!ca_!gEq(S=a1mPy_@Y6!Hka%zg8 zZ2kq4YUyb|Ns<$)143!INV=Ad6vT`zgD>%u;%)4b4ZbQz8=Hp?eRa@gWfQiJ$5}4s zhMWJ0Xl#B80%vvmIe+CJ1u2;ria-wHZqy>QA^7YSY+RuUJm~j&I|9x*rXFVuAn&10 znVB^mS3F8+jZ6t`TEi7Jq!7#VRYld9-B1zg-&|SM=h(bcgbLb*Wid^XzTQ>*`0@*+ z)XKu2CTN&j=D6|Dol2QVs~>0m-}^FQSG?L^1XGvYW& zH!Nb{Yi%o^wMzsw_?B2D#wZD;3tT4Yae?cmK;7Y>2)xy(2(Fqydl`X*uM)ft98BeH zcvdoPxZdhKJ5d6^bwg?%HBynvBVA*(ODfeI7GC~J00(}pej5hmW-(Vp0jfS{9!F-i zi+{WJm`=W4`2)UD1k31#c2^xqXjsODCy{#SGYVT0 z=AQ)!#p8uQKbO$h$d*+XiOWwM}fp)a<(H82f*{aJm22`P=;GlP@3N=KlPr`32*DNq+i{ z{kNmIZd_vh--Z8mHotCU{x6ve&rz(kl%;)aSGp1*+iTyXhiH=#-=F;mvWUBt8iUYE zD2=Ufk!45~;tPIn2#Z@lYh|Z`)NBGIC~F3n`-dzf<#$$p^`YUezFrKm^d;%lTdeDi zL&rq9M^>NGcrnox-zwImOz_~m23T{>s#N8KLH@>$QGYr5uQQBFbSQZl_$EdK>Ml ztkxRQO9-HdtzxxF3N>p4WU)7X?$4svjS}Y}Q{D&Zc+%YXPE!Bqf)4?>OkmyA6IC|g z;Z25BcQHLup0un5QHkyc6v(BWg-}|XAK)f&B5RyA$GYpFmKq~f9>g-`;S{d)A%0TxsKhG2<*YvS4uf5QPn~_1Oe@pf z-CsB3(({hnKmQ8XG39+_WeM=1;W^e>nYP_}c4v>A&RY(KTFJyQ$%=(n^asN39B-3- z6r_bkZJft;nPDlvmLRndY(ZYP>9|U>|hW&O%4{cpk=M+Zsrs8ti*;iBS6cBEiz0)a)J1g zxOpGAW#~q;#8q@+bEbDwnrDYrFYxKhHi2D@3cc$H-+{UoB#CNOZGSI*phR)Wq}J~X zM|IGfh?6j6Z&Svuvm5rcZ31W4H)|F4dV>!?aKdu>w|xNI403qcMpXg?zK8ak2m0OP zz#zRElMWnf-CYTW-X4f3M%nM>K6L#gQaxk|@FGmK0UfH5g%(ZhOjIbu*3zIRR6{61 zjB6U==kMk`oG*uOD4*eiZu+E6t2KRzHIJ&^sLe4VU>Suu#4gtJN5IVU)nfAoDwL;1 zC;>A2#OipL9{6_zV&N2=XLpnW8@VIp&FGb#*eCcCH!c(Lt*7q__l#+H(+o!?_c$%n zjpIKrLM#h1wGnHn{eYMsAIe$2XItJ@yN{Rrl**gm*hdl9@e7%jQs{Th(e#Qqx{$q# zncjrhu8$`uzCDf~JFW^bU1zqLomKUJnT647t=->3IElV=q)2(z0i9Ya+{p-J6a^b$ z4@0S(?Mw${i9^=p9JLKEEoHzXh@kKAhep=+_Tz=eW!)+oZr7-wHf&0T@>S1j(jkLx zn?s^DM#Jb8T8=>@GUGp9n1k&isLy_(P-CMhN^eb0+0ys?TB(y`u8vSCbNlcM1?Jq5 zOSxRZ=|0C4Dp5BXL48w{#;%JI>^pjKH&TU7vw9%NvsD1 zeUxohRCSR0sE)j+Qi2fjI84HV_P0(BXz1)iE4Enf!Egg7H{dM6sUR89*-kPD$g7aR zzO1dcAoU0Z{jn3%kXM?z2xO*2@*NE^S$n;{@D9A!_YdhWta0S0;K0S$>2ypom*d#Z z{Y_rRZsB6Ry7sdFw|(=s%lpPpMbWrsfjk1KCn}^wznP^O)~{HBwhe^YpeW-dMEOp^ zWns1jq9f*QR1c6-owQEE3wwb)sMI@Ex`p+@9aZ%jwO?R+aJl%DuWizI~h5ivqLz56LupP833j!`p#ZJ%$e;wQgl}<>K#QI*gadhOW4UA58no8-kci8|&j1+I1fs$G zRj>%0cXcqGzomO_h&2cYub*`}6z9L99zb{7-*}h6-QY8odisC`E8-#V#0s+^rdqrd z3xP{F?yIvWZVXh;54~KE%ZKn{fL73MT%Pjy&KLUnsUKi?mi~wap%p{nc5?WW+cw}m zK;iZ?&PO8SGIT+RuQE39U7rJBN^*7fEnbl906K35?D69S5o{Jn6o*QvnQTPjr(Q93 zb0cls?SB|4dKV z*=gYxO4|VdE_7nqWsP}y2ia8}1?N2h?hd-`=?!Jc_rENX(ykll7RqBg3qTJ>ly74P zk>~`>Q%A*&Ycbx?nW^VYR1p!kt%&n6;}2jWT3un^ZQ%vWT@~fZ``Ga{fK{O1$6-bOg9pe@3H$; z(7`s;gI*2zM7mGzaI_zQ{N2uvI9g6jwmfG*EnP9Ft39ygH#n~i9IKcTi%s@DS|#c& z?O;(J@U!~B4)kwxJA&4Y=c~ltQ}k|#wwg0EVtCg?TVKVUfZg&0*IK@k`P}6$w)FNn znU$`(Mpe@;V4Lj-*x9w^7np^pm`CWk$brwg6>b~Xzbi1!3ms=5isPka;gInXZ@IE+vkj1$gsPHSfBKw(R0(^RD*j$qR zCXDeiGv2V8Mo+{DD9vw<5e7D=-yNL#w+JPe|vs8R&J z%MmUL1(~jM+2wjHC0H>hcPyy`U*E~9D5xJ-xBGj^9qEg`apBa+F644+f_@lw=!VwI z*0Bozh*2|L{KK$7rh`aZ3i%_N#Tt!abr+YlRZ`_;AgL-nYr!xuzWO6a?2>~AM5Oi9 zDwhkRr8Zx56)ku{VY$>tDv~2G-$js!PIO;9rU`CF$`(kBU1j3=OJNU5OUGy=x^-HO z*3y8|Se9Wextp&=?r6fgG8I{xL~hD*VyXB16V+7A+B*jCrqiqKLNVIk6n|IiU2a{Y zLrq}r+;h!9dqxcE%MYu>{Rz{g*W%Whr~xQPmJE)sRR2PwkXIDX8b2y z;<*>?{`I6ZM^E%FRleB`b(Hn%9sU_h>jNpe9A+rVYe=0MXc6S^1>u3Cqf{eYA0*rv zNvo;2JdV8z6`g%hPQf#*PblVYfo&-}=y=j&FsHAZRNuds87Uo=7)_a=;;E`S+-A1{78gdb zmZhQ5R8!drArg@zV)#u>H2ERWf=DHOTc8gTRj0|4p@f`HyLb0e*rkFk2+YLNll^_Z z6KF!vO~2z@a4Nwi({w5yV~j9kP${Lg+nd3@Dl3+pJy(?YDL>GcmCl{OI{>UKPK^T7 zm$S5PgKt=eAvOOxObJToZX{yU4H$|oTMJ=yb_Ypb(F3nskeKI8HEcM|s}-Y(>}ibB zHL7!_P?$Z!|8c?O2{iYSHPE%JbTk~Be~8d%Q3%0!j);E-X>+QKGZw05$_Kpu9SoxN zCg&4mp+52pw?n$>G`5#933Zm--DIiZZ_eH>m;qng;Ej2js&HOB+MgcRyDN1#)>tOn z0IH6KFpS`rCmm&TO_#7->fcgno$*eV*nmk$5InmEPj4 zh)y~Xx<^%GS-3VZXou2epp&5F)8|HZ+-ESfCBxfh&|eL6xVm1~?6+B4=Ew=tYdTH| zK`6DtDzlrmZ*kiZO{tK9;ciQf)#7+eb#2rCxM{(*s83}pNk0PSzy*! z-_sW1*w8}E%Sm6NxYXR;oMdAPPECYL<)qzN=)9^MaT3C@W?ZfrQ~8duI;bXAe{ExY zX{cDgT}3qY?lh1kgXBO@-#Cp(I(7?B&G~T$f~zF9GHmlK5+8Gv96`6DakOIi(IKY4 zaHTb^9{{uEE@vqF@MQdYD>CP1WVRLBMmNhq3bj-%EO((6Lw;`HokxC-bJ^?ic>sD5 z#rl6ZKad37g?z>{H(4Y^I5KYYR!nPI4ZBTQS{!oW$=?unQF5zP&zHDijn?(Iv)G;a zUWdhQAv@$65=3OtL8ZfZ+hdI+1jw(kVe~UkWZANJ#njR)av0&gC;d&rXkSh~q`F9^pG|-Cx+C-=0p4 zU2;oXjdchoy2qvD0-fK334_V!VLw1nS-z~sF3|#I62p!wOZ2G8se21200^~`S`v}L zfjQ{0<-1a$#GIK;p|JgLZwwtrl!r{pFVg**juEDJ&XvVWqbY@)3-cQjI;O;2|E58< zzEJg|UfD!6t?nsmImgVn|9dixl5)=FyX7*A!VrSg7q(=$s?%VYfH^RFeH;by>`@dW zcDz%)nV@%^G5`pD>}H*@CK6g-!snY#Fvr{*6Tde()_z8DckS*O%2&N{tr>%KQ%6s7~Q-!siskDK(fBx`HJ5IWVJwivxLo( zZ#Z>cd$-^ahu47>mAnbi8z)1_W-DQbAyl7d%z3iR!T@;-Kkln}7IbZy$<*(GOQy<* zeoU@%;fzA)FWYfKNANZ1X0=FLHQcSpRI~;OW@{gBU216z9*T!BDNsGAQB`$66Ou6n zja|#{{Eicn5h!^p_vhtv59F;Hr=Xjga*c_LvDVi+kTiKk+A1OaF}{k^gPOcRvEmtN zf5lx#V8yTgI;C-l~e7BwtkmEDgxg+A;T;zH8+}(8wpK z{~u$3k_>5~p=LreZ2qR-iK;Rey0FAE!*+5$z;5_yIjm*MZq*Hl{-xBV#T^J%6%}h- z;JOA9V22*N!Fn$t1EfC;WDYJBDFB1kZxurt;8=igxiJkuVaz8})E6k$BuwXrE^rIG z!c?@OH8UK@Dea5SmWGKKSh51~!x*S+MWXvP22b3(R@%VAGxp|g@)!lub=Zs!3nw1Q z&`FG~%zYIJjPaY6xtjPK7ZPRRGP@u(@p(JI+KiX5am(1ieg?NI1uS-4k>E%R^Lk); z8%G#+F2TU}^x_o5Lu{70X^2x(-#S<;#bYEK|muj}>99}n-_2|M^``xH;I6Cv-j@Bc05oe_ZiQauB+7UsXHRyYpB{ zS1*PCl&7aVDoEQ~56rt6mh>Dobp1f;TjA{`R?}Njt<-X!9SgAgc`-A1va*^l_Rl#C;uzLdSf~zP{JQ$GtFO~V5gERIn^46P&o}Be zhOL8R%l@})kqEVQzDkr)JwRjTd0R&~=_ER37M4VDrd7@~Zp)s};b12i2Z7(v{r)*x#_K15cb|55s z-5b5*>xVpG`%LG2rqzA0(Aw8%olIK)eRY1;*PVmo@9JU_t^dWt3fd6Y$K`kB_d0cd zzpx;`$NTr?^zsnh>+^;DM`Cqa-u{;#5qh5p5GUWaRad;)TDv;66SGi;^2|aveI1T{ zd4%!A3?9Hrcs-FkoYET1E+9Em0{O?{I&8C;mT>XpR$AK%&aKjn`(eJO5-I!)sQ!l3 zT|%3mSbt;s!m?l&F<@0RcF{cn$M^myFfv$Fy8w$6z38b?Olq*VFInm4i?I44navF| zjr%)bPPx8)-^vB;#UGBgoX|J}wTs;U50xNDOFS7f2r?k&NCbhns0*jFWe!sVY=<2S zTWFZb6grF8-XqObNLj5l`#Mx>(pDAPB{%vF&7H z+qNdQZQHhOTN6%f+qRudZ2ywYJkP&scWZ0kbl=DX?am8?&0Q;#32ae@GCm) zy}z(5zj;zD8X?$}lASE69tg@ta;)WOK4H@oWNu8ImU(b5j1_AXRYj^HnOa!Q`4GO5 z=qd?QQSdCqy1TKI$TV8fR*I;lrUNsvhkt1mtE^GABWuu9 z-@tWHmabW0_I(-58qXV>6~)U=NkJTq9LeOQBM&8riR<%tR)VZxfkaMD>tq^LO2gb~ zu`*%&DoSun?(4mR6Q7A!Mu5tttuI(pAmRv$oD!`QX|5`=$;tA5sbsLi)7ymnTnV43 zRFhYIj4*$=p*Ayerr;D{4c{~8Ogq!0*8G&McCqeo{3C$lm2E;I>fA(Bu3A+O*Y5-- z3U|<6Yl$geKlk{D+&n@f6A5^WqUvo~9#)6_1n8B`mwW>ke{V1aQ5Dya=_0QJ2c(K9 zup*zzswNZNc`l{tCqQcYTK-#l1O5VYdbty>Bj59JQaqRx@mbFUXnjNZ=pRWl62-p@ zB_UMZv~&3c@)UJm9O=z?j^?I;=ePl;i%siluw=hHP*$? zB#H=Z!>je6A^+uA4C*?xG-+3GnDYOl13d1IM%X$9q6am9Jp%P=;}5==UNnwd=pHN7~`F@R-;@>?3Gtp%*XZiIvA$5Z|r zra2r!MA{`evB*C@fK+!osQJub>;K^c+_71*7=yb%zRMm(IUP1Pk$?QjcW&X6h;2VX zBF~yk>!LIKXLsWu+!7e2?XVu1WM6#;UKhKffJX> zB$e7lZ0lTKY?BD>rib%~1EvFy;z*noz9n5wwe0MEXSM7a*DRpOE0-`cQmnW4Ft;MR0K}!xJpuLic5hf1Gmamoe>*+MX81H z1LK)q-@L(am_^KgcmryHkcWLsN_)mAhlbTi&nvtH7ja$()o=Bo%Nny%2)~a3eBZAy zPS*v6!ocjDZf=0{?UvN9*hq~fh!IsmcPCUgb1nWoW_*$HaXi)6n3w?OJ6Z09aqDy5 z>l5`-rXaX_`Zgt#=trTn_A=h`hS~HfX6f0-T2?{t~HZeut$M9O%r){;B7)l6H5uJyFVA8pkW~qKtVJ5x%urOs9 zC?TaJjUWNZ@U(&sxy7QZ4@>zfpUA1^-TJuMcSTW#I72*l9GXPbri@v3v9GhY?Sg@+ zD>VSoQh4a4XN8`0`bJL%ivbpL=%(Df$jrUfW`n{XG}4Y)#~Y??NB&(1-f0N0q}Uri zmK9nF>QEJI0@87fq~dEdNRj#sah0+4H2|p68Pp3_#g9jqT{D$2*~w82-PU?Iq`70$ zU(&k}(&XlykF5l#11(DRM|qf~T8d4>mZ%r89ng9wOuRhSr6cwaDiSCZsN>uMLL0o* z!Lmoy1Tf9}LTx63JR;is_VM;A!US;mfe5rf|6qgBv}VY~c9KM#(!moyE8!j(tE}6GnW~)4?>FqS;#KFzr0CylE1A zM*ZBo>nQL<#kRRJ@XVdPsi%uF4}Q{I9T}Xvl(1h3{3a&ZjHW3g#>x{UWwwIE0QSs^ z6TmF;@O^!>&7SZhOk4iS)<_ZMK>)lByh_D|DekFLV6ZaBoLDs4X7^Fq3+LZ{0nU> z^1EMDEDPJ3p~w%S^v-cA@f=~<&!xC?zmK|RWyvsi^dn-;_~i>dLz`!;e7=fBG3ZF6 zVNib0@LMo-a03=-QVbAM)!pY@Ww#FOspiMjwb>T3W*RuB;9d14YSWMYz)sdF!ZZ$+ zyL0$Rs59dfqsO}picU1z2gG&n(2E^L-tvm=q!D|^9spu$w+%G*eY=N$jqNddM?XK) z$KKO!qhvYB@ePgKkY8=3am?c~^@p|6f9(|!f{4Gu8UaFxy+i7J`!VtNd;vjoN_fQk zq8hiu8aHAZ*JAI$5(Fg4));~~jq5RpJYxeit_bzZL$840w~yI84BRJmza2C39>F$` z%q~N$E$!eOT`hS>?kn0wsg1clD=u$pF0TTYbIzioF8Mx{z^>@(3kf2Hl4nCE5-+t_ z#zT(_!w;Qpr z2AXQHdLu=+sbBs3GQmc7JMx{cN1(v;kH_bUWEUU0E%DsgGPQC}SW7rM4Cvu#jAezg z0f8dqWXKj~zPXL~gCa7ETwHn9g`0|zmw`R1y6oH%#}&!d90Repl| zDU9Rf7CtV~5+*a@l0d7!v(h6fPJ(JFe1ir_gS-(o`P|aQ{@wVAK_+wK1YLXT3bR$U z8zSOD$WR5+DZ8F1Oof#$B@$G2Lv>A`_M#LT3^H8;#1qD-27z?-hzFsoffLh1L{efd zDopi-nK7xF_E%voz-lfs`;ruF6hMQa!=jN~JZAOtUJD7eKip6V3{DZQ5j1ST()k)l zxI^y!JVYtoE8*5;y`*lu2+Lel{(%OMSNjWD3q4)F3q41=3&xiR1JFVNWk7$_7jVQV z%0a)(OaKPrtW&_xN-V>+@^^tmjYZHsy%=oE0#x9mkKD+q7ipCC(Cid5K{OD3aeuaZ zy8ZJf3xl)Gq9y!y3cw&QE=esX_NK31#HA&+)pqU>&H}mmB3G+0JR55$^Yi=+1Lk~Di0oYy zG}+~QE*vV)Qq$IRiU&^***8pi^{U^gN7jNY{kt4)hyYjSW*NoHHeYEiJ}q>`(1CL4 zqjQeijIKQ=f}I^-c1V91`vI1m&JQm|ds_fp_x2Orx(^eqaAjeqtNezSMzi( z5_+;NTQ^a8FeF4YtJb|gwd+^bKhU6hbFJQQxh45AUxCEMtr4P2VbnQxrQ@&qdM)X_ zl)vH3ao6 z6wa{D?s5an<7MSNiZE;ihju6IyP~(k%kJHgN4}8f;;E73L(^5*r_|e?iefX3xLA7t z743@kf>*$hi=8UHqw~P(TDI#3n539e_sFOd2C?b_9-@pZXj9_RU1$kK38$oVamb-q z`^~|;S2_GhrJTcGCG9uVG`^vd|NAjdOM3w|ZF^^clFlbmYR99r_2K*JG-H3htkwBl z`k$HS7x%X<;@pmP>kw*M_kRc9rkRII5O{icZ%remu08*27AckY&+gw2--GUEyQE5v} z4nXhL>>4LM=10hG7Dq%Ex{^HqiDgfqbmwG0MURuXf#2$>@2ek(fc zN9D4*&5A37Ehiph^hFaqxB&x@alE;bJ-J;^gq<#B*BZXADVp_oW1{t zntZherEdI}&%>=7|4i(cr}M+W!djl+$JX2k=f?=fh#&9g8sNZfWaJw{@Vw`+Vb>os`63l30S)8=b6Ns{)=ZW$&!{biwTQ> z;kj$vI`LCT#W&?VbMPk(ju-RgcSa(Hrn?5|A7q>{7tH3kj{5mI!bU@^Wc&+F4Wl0sLDt>jKoEpp&TXh^m1{kV) zUlxW;+f9L`%Gg=P{Q?F%1Th)0NKHtru84jckGeEaQj6}y5W~IlZcAfBbReBwKzZ@@ z;{JvWT+;iM&WClhW-n|0z~zw~O_UVC|FsiJX=*cMBU|_V$e!tU~ZNOJ2OLj zWcvJtH$0M1^ezOLV7TUZM(A@FEf!lv(ZqO3S1E4adH<0CQ&RZD^oGkrXkWcR0CqXkOLGnE{|k*Hm1W1wfUG{Ya{(_pEnX zf>I4Xrv12oVYKcDjZc<3_q(dU)@e337nk}M<=OvKdo^qs7h*Xf*=Ta(|(NDt*$ zh&T#*4{^fU{ciN)%|>cl%86J321SSWMjUtaax?roA1gy;3u<;V+X-%p%xabmscebs z-P|N78D*}plO>r!ThN9NqEvQE`P3ks5#L2U{f*;i1BJ#0{R3N2&aeC+F_bd-B&0O- z!*p(f%p!4dk1DJC1L)MCmwnf4tyL6(Z}SIKJDxn(xuWlT9bvgAGJfjD68;A^i2a5Q zWRy~ykH2Aqc5%f?De-M5fyDb~46`pM>4`zxaR3&{0W;j>G%o8Od>Yq2mIeg5#2UMQ zumM7fIdLm1%P%@5Hu4h%LG?>|AK(t-L2o)1h?Jou_8)}IK%BEeM0Xi0iKZ1{6NBnx z7cOpkbSNYDQ(Ld9sziKLEuVqK(!Xp-#TT&iSyGq?%|oCkbEL}5Z%spPsN1 zo3L96JnGq!RoEx&E)+|T66aV>rKo@F7Qv?~xlAd31gYFuvJ+6e)&&&6>B6i$K5)t6 zIRW~m6<@6YLljIdTN6^*rInH@aJzgw;PdkaHPbvyC{$FUf(bZ+lM4akQ140#3O_5^ z(d7*D{lihMc0v{laR%8^PxEP_Z7M+6ng9isMtM2R0}x1r9xvU zsf=HNO!{42D!UA!i))n>$s?gSq zQu1Z?P2J__G|bGS4U=f9W_`V;xeI8-$}8N;QvJadk{$BhF=MW`8x3kfBzcL%3nNCW zTJmu^QjIg#=blpHeBo4(VIiZ)STj&aiNC2d-oXxY3}#DJLISIsr*#h3uu0l%F0mY% zg+_d{9;VmU5n4V}>l{%VK3r!e=P~|H{l+XU@7r@DJBT2h5^K>U*%J+5159f zdc~WjPd(Mn`(yt!Z0O;mR=I$~Si&PKi)Iee?hKWa*f)h#MJHBJGMfsA9p<#U{10vL z0yD(GjJ~@-idH`9#$^F&u>4~zw*^ySaybjse}wsm$werbK7UI>{}e_khHZHO=-6qX z2^$%5P|*Ncpn{gG^h5gOu*sfZJl23WDm#Vnegig2Cx@xDn2dECRPyPJg|y1<)fQG# zTh)e9AkO_gmQr$pWF1L@hl8$Uk!5T<$VFKU2%H14*f(u(ZH-b`k`^q+%A}GO@$@A! z&e99jOpmf6G;w13dmTTt1AC0GMdPp*%Pj6fiHrqg^Y+Ft@e*w7vsGeRcn0f(mP{xo ze^9#5c@H*JsC3dP$8htSl*}XJIS^Gxx1en*dxgnHVP!rrNORYpKmr=27qoQ~recM> z*ci){W*x;NhHHz^+H}J_zGh>}c3xZG8^i*rz8_hGK<#ihChHw2xY5IZxZ^p^cb6Lt zCmMTTH$yLG0)IFBEGCF-+=@;y>*Qoc%D&iT3kH?y{KXiy$2kkt%dr0oTh`xXBYeCzBTMIf-i^g6umpH^gT*1 zJOioZ$pNp%QB*J06f<+-q7XBboSgfn-Hu*%BQL1g=w2K1L)JXgsYJ*C<7S#u_!~CZEu=6fm5u?y0C?TwNnj0!Gler3ax+>oJ_j^- zCJFmr)DW}Zsco9398~KMC2s_2KN;NNF;^FSd-gd|aHF}iTB*!X+&zAUvoW8o3x_@Y;r?J-Kz^jxf|xhtCrsZVb}oTSP)n3 zF7YuRHm@W=z!!H}%TZQtfnQgaQp*qid;wn+!85r)dYrIom@+GRMvy5l+D~b6{SRp% zc2Y_@HDyv1p0f|81Q-3(Z^mS&8W z*2(E$&KPc6jp~@drW9XVB;2lp$E3tfh8sI5hO2w^|lCv8vtK#~$ zomp`qt>WsslT~ryKBH^`ky}14yTYup_V;>E*~F}(zTy6R`rAVId-D6$DDQg@Ws|b6 zobUeIqo8qr_#S`G`0p;?#?!s`8z^?)20H&W_}=QjlmGo{tQ+`2rs8�|p41VbE7r zv|9b|)QJFILrkK^c9g2we1Dtw_;fyCmc-AdF#o5Y^=zaVwTF_WKNN*3i4;|oMWNp-p1Jf>v60ERm{wN7m<%pA%dfLLRYk`b{#(h=7F~~zh zR7Sy;GD$A{l`lVk96AJM{Ze?xD3KC#%05jzN;X0XCM&^(beO7?zAv=RrT`Z_a~EGm zfk>c`QU43|$<%1<>aJ*g5Sa_K=%RZHCpN4>F{x;gXq*kVdQuk_UIKBsayrrt z0sKBu=J!dgVLf@dE;R=kqiLDBl$6d`$7BUs1%6YcH9NNoOaWlz0iMa#qhBu^NJ`$~ zxm~tN9b4)wdRpk}3*>G#=TYq(fF`o?h;p~XLlv(CYOS79h6AUktbxV!+}e|2JY?OMS%R#fKdwt`?`Mj*Tii@H;6~>e1j+ML%Qqg3R2{s?c?0TOWud2UH)Igy%!3Y z%Ee>5zemE|qBDp&3cM(GFi{rgu?sLMio_)PB!^NAj&}N>g$){e8E}%e2EtI3V(K-D z0@A>PIG^MLkr99dLWgtyI%TZK?amTST871=V}#*P1&-jJU1u^j%{0DB`TV}E4k+3F zQV4~Q<{7gOwNYDVx0d?V0~HhMzcsUmu3qkC>p(q;t=@llGqk5W%?#h145kBi4s9Ld z@vg*QvJB~W!=2s(0zyXvm&yhN1SAm<%gLTlQz2KY3QXAIENo6moiIHoB%fuqZR_%u zgIZNnf|< zSztN&40`e%DO^OX;=xfat^=`=I21?rLS<<`pbS5b)AsoI#!H)>LI2$cg7_xP+g>2{ zW=dGP85*=-oUR4$^Spxy=(H(?MoqqGq6n0k1w)Xr)QA6ZW@u+MbDQB8!~qsZ`;$6Mpq<>|M(u}chp*Y)r>d6@fnsoB_fy{VmL*6piM#&kPM zOrV`3yKaXO*7Z-;#l`L^6%M{Zh2WtKl)dQp@0$&m2YOm9bvr_j`CN|rtrUa3$8fr=^ngI8T$I^c^N=E%Y5ghNmKac$K_%BD%u%F!FIhXR!hg0+DKkq+fwuH*)Ai6lFFxekcHBldNoGtPAJJq7 zZaB6_Q*!9lvG3&ufxb(Ph#hIfrg^hcYmj->S6e5HZCx=bxS#@MBn0W`axeq*jyt?v z#6AnSen3Ig)JoTEJ0B2TPY$F9TwA15YAJW5UBL?RbA5VCO>Fhe%v$oBDq6QW!hqe6 zgD*aYjx?H{rKo|I%-wMRC!B zr_5_fx}ZXdO)iT^{sQ=}rCm={gtPyh$YqG4!#ERM`uDP^rf)w#AE`K<>kf<)v3EFg zy`L(or1ZzF1zxROY%U<+P+K{(96raE191fX#aHmNb%HI-U2l2_LTuqvB(Zt-s4#U< zHhfXB9f;DB!?~GIf}SSCz)@F1eu3{awtgORqCGe;F~&YCS-F}}T`#q`!TwyfmA|Wi zhk0B_%ga|X7CPfCB>WOWhh{y2KpBaYG{2kh0`lQ?H9Wv2JePoldYSIm3`I8{Nme}V zo%QEOTTGZco8MN@Gre$+gJxu9NGe(t^$UX5JIO%k(4@;w3u=E<(gaFq1 z76trSjOj_x)3V@DQPRmT1M-s`=ehh7rXfhJlqwBP6CKZKf3|%gsV}+^njL=;6u*0~ z6_K&zN!#@Lm!sq6QTIYL4eMWHpi>Q~4P~yWNMFj%KyY-v>{NeaWQ@An zrS*_gDf4&;upIa&*g0j{S;o@5e=-Uoy%9c9>%hG;g4(>KJ^6miqwo{Aq-yjK+*(gt zgUEkPjP?|%gBjT?Zab|$+C`1l@s92=)C|Y|ptlW&yNYgR7sB+jx8xK?d`;r5Ilf(2 z(4ZmN%0p_p$1B1cWY`d1d&vk$(5{tw27cHj4yL6N@ht>kkt#^7^DJRO$K5(V|2jKV z=OXjbjlprf_VJ3(I6Q-Z2!VWD#>u&`hjY|8e}^XhEnA}(ejb~?9++<0U`6@WF{0So zU~sviTeDW3z_8Ltftf1h&w7DD7arB~82YR8(s}HTsNvT+o5g@%1N+&~Rr=F7MlI>o zQ|lWLwU;O|^bu)6nr333VjpU}4x@MfbuhQZ>MGjwSc(;#)@j3eb%~+Nd6Z|F+XgU* zs=zn91T|HthM@QPfi7pzqv%0>Rp-wuf>G7vzu2tUGS_7rPVqZg`lXjd;JijUU~gP9 zLrWN`Jsurr^^}pPfTV6n8NYcIWL}YQp#>d^%{<*k*QRP|9E!WQJUBjc-)7+iv%W++ z+if+J-u*c}BZ5m*smJO-;1tgE~$8Q?` zpAU0vc-Rt6dFf4WOZQtQ4Byi9742|mow5^|Bw|l&&I#U_->kF?qpG`Gw7=qWmnT_B zxi&KCCfZD!TsPP3ohIK@jh-v_DmS@Xx6%Zwe*7}@j=qF^QHrz3A63@)Eo&dD(-6}% zci2cEIq@0E%m%VRVXKm%ey<|S4o~ZC&{}O;0(2jp@RM*zDAk$(wbi)&@FEmRldhG{ zqOH83VUKJTNSqn>BoB?6-CPmzFT)|v@@1VgjBST@srMRkO$(A+M^Zo4(D+)#KBd<_ zt)W}Cd+&G2)6?-ohm|i{+TLZ*>#U>U4$$Szx0?0ZN#3b(M^Uh^h&3sJh7_79{y1eK zm<}@iU5=;HbX$hSQn|)jbR=E8rL0r_dFe{MW_3g&r|SO98VY(3b~2m6!?_}Q2~uiwi?!;z{G-TB)p-;0+;W z1Zr>@4Slg`|2j;#Xurjn1_YD|wlSvc~egYurbw zGHHJEigP%s&W7QOV=$ZU4pwOOo871@?$I)N`pM;wo0egHIkjn#GzqXgDVY7!?I5X> zhpkFJ7kviHmZhbIfGho$b3Iu4D?*L37WL~o&X3TvJoPSrj=$q2*Cx#(z|3}!$ToC= z-4PKHUtOj_0kPG88TLab()>Lb{eA=VAC(7D>v7>{;vxqf8_>4^o|#hMKftZgm_vn4 ztX?GwmOCMV(c}@+59g4nDsV!W`x-9{cI;IG~05GF!b$GCj&#y1(}&1?fcP zVpGejS88j_$B?_W8svsh6D=uCqH&^#q}sw;+7;Fx`p84?`u4TwA;MnHjaabSyfB}z zv+^8O+v;P?vPJaxH^2zwQAb5g6tZ!jf~;e|kPS2T_zj+#(dJFxXy znm#rnKsSNxMGgRhUYhme<`09MxoV0S|wHTbSm$gQY@wFSK`XtVSALLo+^CE({Y`T3Y z&b^=wR}pqwf?lV=Hp~|Bx?PC_NxXcm0~C)M)7JpSCf!cRIOVj8>fA5;k2-ylwqjxp z^qNAM%^|r=jV?bR@j0fsz*h-p;UessGLI%~bI|jv<3g;vMYEr&fr{!hmzQ_{v{y;K z^j%5o>7!;2yH^4TUJD=eF|+_*<-WPz0B8i^1B&e zEU1`|8u)8VBc3fdlFoO&?YP`venpcxP<~DlB!};KX^9UO*3Mgudv7Z zXti)JJi)=i)swY_x;CS!me{fVR4(6181WG+Q1zhby@gA==KOgtC%|ydoj5iXjl6si z<7s9oD)E@ni@L$P5PYInQgW}K_%3P|iV>vWdW#WWW{MFW_>%Yp$b9szzrLG!h^I1y zm(=^ZTnHMXrE_QZ*2U$&&f!RF!(4da+OUAna0S`0=BR?8)2?iI8w}yJ0sX24gNdP8 zLC30vf|2juBYL4=U}$!0g;6#P?|VTh=ki~~6T{Q9k3;9v)3ah@Y02R$W=gECnPTI3 zi80KL9*ME^A%{Y_*_c*fQ&#o z#L>m0gfdcK#c|fjDuUAp<}x$ME51wKE={GWksn$LqRB~{P4O3F zo!(cX5I<|r8?0bCCWuP+Esn`4EJ z=q-*410NLXR(iJk3Qxa_zMf9qP8Ves z_oFwFa+Sy9@JisAOM9B=XBSV+*-d;5Vg2b12U4|1=fWr23^==;)qGDq*z|X_UVg-& zdrE?*+~E(7@p<1`@a8r3A7V9x<*vPw)dJj?75A(SV8TC;m%ALTL{KW$!y$%34pVlR z+5-3ppQbf&aS%0%q@)#8h~ltg@b~y~(JYOav>I!q46&I-yQTFQT8F&w^gY<^ZDBTj zFY3RJ`3JW*4=YHWoHeg+v0D3X6a7{JIb)VL;n&(`3QK1Fk9h`yYE(6Qv; z!>pJND(P3WXN57LKliM{`I$8L^<@%mE!sUyiwW_4t1VbQ=-{sk#5dc2P5WCsq#+Y? z4>7FoGzD8|r`VxUo6$*I+Jn*1_S4pROs0lRv5)Q>9#Kp-^5blJa3AzoTSnZqYS9&v zNKBHRc)rhO!5`D-w)L=wbvKO*^is)Ed-q`iUP^fmWeCjO>o!E@>3VS3aIfpwqgTqx)EY^gmBgI?yFh(tvZ z(ODo7OeML$q~K_mf{JYrxP9PQpU0Yp@)?lb^_JF^16{#NJp^;t1~2yzVfxwgAJtN~3)TJs>>C#bwbI8zhBv?g9(hc0a!Tam5x=>tx8>|&?1E@KR=9`^_o@?iPZ#6J?lT>>7!J)``tIN)IHcBqk9U&^@U_74A5$y-dffIQOgM zL}HC6-Wp?I;$PuL#8d63E9l7#I*<7W_@?kJ`3qsphxweyUZ5EDVIG6h+aALS@P=9=t$^g zxL3Xu8aAqkz(P!Ins8$w+K3vZqsL0!B1l;F^94p{pV}uH>F2OCKP2GI2kY0IyO85p1M+ydU*r`fq z+{=A*s`qVmiIl+xhH?WlsQvnRGr{bltVlQvPZWKIgc*ofgNI+~ANWaM2YSO_f@iQZ zPMY%u7eNloddNK&S{GGYMU^%iB@6_j0j%b#L%d2>*2Y+*Ux2UrOH?b9C$#HBP;zWq7F@G!_wmj_#ggv^HqFxwrcHY(2o&k*7 zM&k%f3;Xpr6{*CcTtaiaur|=_=T8ER%xZA#iN9Mr=8m*!6G$5RS4o#cR2g*aBqmsR z$;iY`i&uOCaQiP-2R48)*OAWkMV>HG%xoWAh#?=DVxZ`D9M!;$^puhdemdjHNxj`% z4zr^nOo;7`OM0&`Zfl@=ZshMjjJK8DD9o7c@;B)K{hJzWYydVJ!VPB@O6G*5KyGZp zBX9^$#A;ZdpabJu@M2!80q`;5lqPww_tcN&31!u&$sZy0II6Zc?_qhgbi*}C?}fVL zbbJjif#n|beDYJhrE-ydtbzt8f3~r7yQUqY zTdlQ`jsyA=NsG9W@laHmLTGO9dJ8f<)e-3&%F z6SQpF794CtdmTsroGT#wBpjg8+o|guHTV0F+bDa-7K%;Ar|HK93y-vzlKtxM7T%?+ zIG-`2d+8ruUyS#kErh^dl0unNy#@TSSJKru_~Dq_Uc}Z5IjgSJ|L)=e`c9QX1|*0% zJqb4RQ0wMnPM>ArBhK1m;CCW9u@vw5QyUXFc(lFL`P;Q6X+ytj_8V13QkhkJ#doc` z03_h$@>z$zoE=ZJQnoStSdIG>WpnDEG`G zA=u1;7|cm|+ReeRNTe(7L<5!cy6Ck=Jw3d`IC}bq5=?WiI`qLrMBwO33ZxORHy#r7 z>*o0S-Tr+znpnZW$m{k1=;c|@>y^CqA{>ZIlO;VG*I)UNCArvX#TnPrNw(nz$+*N` zuROtIHlVu4lPc68Q@yRN_2{Wr*{I5LL0!C`6TQ_?}+m(V4#ckAq;$(kHe z=s~sviV4}w!*dPeKxR-P^piy(iI?)?A$m*Q`=RoKU(ooKyOu*SB%+ESh2pJwHT4i$eXLuYnU3X zn%K&!wen`{cN@ecv)ehm^jiIp+3wzM{z>@@Oh6p^EYFYKhEZ1H_c505Jkmc^aYfwK7vUBcL~*gTBJo!JszHM^s&^7>#B)QjQ>? z9?x|BtQ3Y_hW%S;NJz?j$!3knVE-}SvUXjCrZ_|K}yszj6+~y`1*hx zF?ziZHVb)>mD&39futyt5>U+Qp!M~Jz}=B|GXn^T*uP?dM$->#Q$35^Tid@0NYipT ztL?2yE!IC$Oy$&&cY-XjuT#B%KlgE}gVr&{2Wxw4rclq2AF9z;sM#Lfc-xsxb3NlayW7_YpcwyIce!|h%+J-k+~-T~hUOiEgiPm<>Y*xaz@qci>@ zz+hC&$6$OEpf~(vc4HHIAlj+?7O8bU+Z=kC{+`6G4zGUV+UIRRWqddz=Br{GjPPxf zuB?JSg|jJrLFWg~iP2pVYFs;QDkZ37vX|T<1;1Hx-|58A(BT}RsBnSvIn)jgT+k%j)W}2Ol*ovxOL`5 zAWRHJfKH(83ioTjN1(-)LEVcitBJJ1>t$ZjGVq|oj2g7K?ds7)(tcPi%hIo z?qq56lYv*WuRrq$+f?EadF-}Ha_pcNs9(z(Xut>Vsb?*!1-+)xu5I}3-jD65QiKAY zV#|}jxrJ!K@ux{8VIeBb%#6fHvq3Ru)!M8`CpZSPg=PX$r5Kr#T7_uhCgb5O!GN(| zj(`2@`yhN@hTpTQq*Kq~jcsNVB=K|LX&1zam~FGexMEx&r(qm zYf$f=sANHZw!%jZ4L#nk^MSH%OgZnAgTWPX+4c57NW9F{(tymNk2Is&xyQ}7Wv@{_ z7Gf3h{Av9nz*I)cA(W4g*%Rl~8A#`bvzHtvOo>}-Mu~x3+eDZ|MlbH)8fLpJh*W?5 z11)7Bk>jNJHxx{Zg~cH=BQfTQ_U0b~<5p%ZW*!CXwt4tEK0Gh(_AZ^(G;E=Pa?Nr& zo(H{m8-KW7j+O_PUSO|aMtRG|*7qlRlhHB;l|;f+Y`jrs2uowQE7VBX$va0mq{a-5 z{6wY0OACt^a}qs4h#csYL=zJ;&MSG%W?aigP;~shI?e9dhn5oeL~%2_)qZ}c17tv~ zfx;hnPBb|UzG^jA>AajTQ`WkmLTz{?#5{J6T2(MXw?VCRlekLlK~{mYIo=)ppnG(* zK7b^0S-=Epsb;{f23U-k&RyEBmZT@~Xd$N}S({3HXVJ}2`hfj=$Eg3z!oT!A?XpZ9 z>Zi7WEf7#!I$h9b1qx{+<({N~`&wPa219oqF@|);71LzZmH!murD(9W;M_LmqK{gW zQDo>3rei;@;r>=8XS^+MAoL$&tKh@0(zLD!-c>%wt?^a+qBG^VT_hOvK>ti|f}%mI zn0T=QweU9eaP*wAZo(4xiMRF1K&C^L!zvIysq%OV)Mh8p4sA&Pp{PEQVF=ISu3WQL zYSsqf*&qdtbIq1dDED3C4z`{YE$g}Kw?qR)P(O`n22XAJw_oM44Ah0|lRe<)?eMX3 zkhjI}>*nqKtx{&U>*?n40}%6nzCtJ*t-Soni}Th0;}=ch=>4(67tqPNrG1(~W5yAP zA(F=H?roEIfzzpW69=;_HgZRpd{;QXeYx0zszGKq1&s4|$;OMuk?J&Z>TK8)F35ZS z(hm;zhg*uQ*X7El6@IorL(>M$@XEC9`B7?ju<;97vERyK8xa$`WFD>TX6;|ToEWQ< zx*sT&yeRtb{apJO)8TpAW;pZ%xJBT$9)FfJrHCVLI!6V{g1dN~n2^+sj%EI!RHY33 zN3I;wZFS_tr47QLT0ei$X)LvEP_{+;1o|iG$M!R8ZDMune3f!DEFPN0atu>va*VV2 z^_xjbfvL&)4(>x!$sUxR{kNN{E2gqo8y)+`vAD)=x5*V>e*2#CU1MtC)#%3-QX@*BG<;v@LO_6P z9!6_#Tx8o?@H26eKm9E{O`)jkR#W;BAKtRu2p)7yt1>m32^W*S(f7?3x zi*q+TEV{}m8UJH3p_95ea5GICy2j*WnLVhFf=nFffGJb~80WBu&V&IPrdEv=$^wo~ zj6kfCv-XX1k7&r-c@x3j(!O(dupg>OqbF~=+vZ>>UXp64z!r8o)Pg&7M>AINsLY`Z z3vT!e7F7^}xOB4>AE+K(M|cLI_1<4qlu1yNjqFKVd2Y(}VIuY($JiS4J_wcS}H#?cmuw!T9h zy~aVYV~+Wdt2Gl)BrF5Y6xJrPvJt=Qt4=O!SQ*?9m8};azm^{eNtbrxBx| z9CkiIlX`HyhnCeZR3|b{cNP&34!a;2ugb1yQ)!)1gXZA@yC&ar)T@^Cw5x4q8ugFxvCg}^xcwx6G})xIZE)=d{0jL=?w-QEdMKN$VLqZ|yv8BN}# z8J^^tJ%U?h+W+;liY9Kk!;0yM=* ze;D|Qz*W~MKG{l(fjrKh@;ypB7myBrMx7U!L2@BUL&^Dy!NPjUH01`BUdPmbzi3YR?VCo7CG@hBae@m z9cBlYn&C}+zZb&!MN3|* zi{=6;8~KhxcE=z)gu5Y8&MJy+Ij)23&c{MF-+mN^IofG1kCVe=49E--dJ#jQ4I?x}A82w7%ARV8a= zkiJ$9mDNl;C<0`I!f0`>oq$s^RY;-uf@fUUCXh-QJy#mZ>V~JiY0C7LtbXD}!P5S5 zWAwQpdgURJZD^j&S3h;lCsdGy3u?jE^oHXt_(f`jnS^SVegTW*Lb*$p%NxiM#WX;6 z*ERT@{zH~7rrV7&sg#y^oezSlIdI^)&D}3qBJ-Syv{PZX{vO;e4fAfDt1H5V(u}4Z z@w$`BF{vDr%Hcng%F+8wDhHWVjp1I@;bM(D`0N( zmk}nbMpxN|u>JG=X@}_C%i!DI&x_V#w51a_O*KqR!2Mn)mYGG#nJ#&v=H#ND;`+F= z7vu|a^}b-@ExvWh?)207)A_S5b%%7k{A~Vo{`AWH(DVrgvGq4c_{*y<4{0BS?~H6w z?R=-J|1mE@-vh1I{qFUj_1rS!{AZTbjy85}J6R-shn2AE2*M2Zy7=q=@%R6_Ht zKAkSznrPnb2A5Tb==s?(FEAd2m=>BCwzSLok9b2XnlsPm)(J$oWy!!R>crylsWn9f zA5a}Y!MD*fgXi>IrfN=J(`4;@^rn{pkY+!>&{VT!m0d9HY5ar&G^5O_POq*RNlLD8 zerA8M|7-sGpI3N`yYZW=y4B?;Oea3#x}D=EEg~IF+KKd`x`qIQdO6}f4-~{-alLsd zg=TlbjQ#f8WG%B)kwT`kjNO2mkG~VDb;%b1fY1$Cn8-Yr!V7XzOlvOcOmrV;!_1Gn zPtIk!)rqk_-lh7wlvPjO!4}9o!g$aOu2>7tks)-V#UG1FCvT855J0k!>6~21oY{>C z%cBq!2L}61wO|>gUtPeok>~i9VtU0>vH%lf4^5-w9|?8~(P1@^p4;)Zo34$zqqd>r9-_a; z_6t_#93h4m+@8nx9}ZIluqbb5sno#w^-rpzI1fSEQSD_Rs8fH>nIxAZvz>Emxo6j`Jtf zcUDo#g!SWOd~4LB(g=`wbAfTojppPn-4L27>Bi2*_We3Hvpv9ExgZOv*E`p;S;pOq z(lhM!(R6Ft(HE#Fpp06Wu+-?0Hb{@^Fuvh^SfIb~kHqU>YrK7LzUaA=f08;9IS`_3ha! z@{}BzejURWSyB$j*hKpWJ#7ZQ|!brG2l^60Dccucj?y^1foyF z07UA&PeZfs`zeP`!;rN5!E7-SI1J_vO3U;*Gx3eSo*&|(KkHpUud&5O__Z~J(j3XHo+Sl+AR&BV`R&7AN zCbfEvk38Hj6?j|ZWNS0JOAyXIMfhY!C@HB(W!^(_yJu#704m&DpXm~P*d-cvaD!qW zSw-&+7y-DdmkzB0=<43mmxr8j{@&^L|HU9C%c0b*HX^pnY~c>qdGqnF*mrm(uJ4zd zVyo}w<6p7wRqvIy(|2?Gwfhd&(XZ9_fG1Y5B$0V>Udm-}R^V=#9sJ0qCAllt8D-km zA>EHy4`!o#a)Dks>|wk?CNx`1r3dF@XHK?1^WYl6Uju&&Cs2Vc4c@gyAB9XAQEORc zshJB@MXl!z-m|d^12he1ZYm1-bhRY6vb>%{ejw)mnL#m;4o4OgRIg8Lbj%HN=FDpp z_Yaj{z{u?r+uZ@8uKqyN8@S1qc<#9(2XEXQAJ})^y}ukDx~o}A#l|M>8DhT_3p>D` z*Bb%71J_)n)+rM?Gd1)p3NTG*5gt8o+kF(Io*pu5_S^v3rbI=|p8J5}j+L0M6_bYjA>GA|t+eXgRIoOvcnJDL8GaO=sxH>e(zR?Vf zeQ1o(PsGedYbw8CCxqQFL3rQb^_p!kx?37$AjzuKb&X@?nqO269E@p;DUTVhEifER zrZy_keSu_cZy>%2UO>-9T)Tr)M+;*whL;tj8z{I09-K-vQ7w+j(YD0o_+_832q3F# zGgn3{k*uRdr~&nEKa1Hhm$lD?l5189mXU&%G-n#)^am>hJx=)@fUmctEl%0cg%%%L z8{LiXz`(Qremtm}&TP;G@W60RyQRk2alXfo+zpe|4)y)ma-0BV4krjf{zVvPaC4%W zEv+{bfN!@dpP8))=7(}wL-?j%0>t9Ak%uCb2~QKqqpVWys{uWqRmg zsSf2JfIh#bH>`<8tHr)n?~ykgVLn{H&_kp6T0a!7(8VmJQrUcfp{hJb4rv%phC9rnF#)L97FIry> z0OfdcTgM>p4U`ijw2cMI8Gyy+4x;x0%4x&mbcx1efpWHBaXLtj1`D$f$&7Q1?pS3fJR0MY?40mky%fS1~;{ z(x@#MV#Gcw5+jmuT{FX+ii&30hJesICuhQRM0KC2o(^Dhj6=+-g}sD#%pT*Sa;fn2 zhVqOqI?ojV;M7jUy03F`Dak)8R&K~DlMB<+6Oei@*HlPF6i-<~%N~pa`;z&dn?)}{ zIM4Aq6S}0C7uNexkP9=m?up>4$((b6sMfjeG?uGRpJfTwu=cg*87oS8KSqc68fTAoKwAy$(MgsYi}=z!p}&hOqV+J zI7*h4uFAgqpkx`NiVdbm_xpH39C^iKG7t}6+jqZ%tn>@i67g`kncRY283-}*0*rV^ zVwAjLCAq3I;jJYC^Wk4+l;@W$FO2$f)tiqvuKJDL6Hl-S_2fTX;(YhKhBcm?rMw%O zscUZWZC_!y1$AdrFcgq+$Kq=P0-6i-rpYx%FQvQ$Wod)AYn_L#$iaaPv(Fh7Ln5qm zuiAL`!^0Y%#rDU`ig5O?pd@)@NgNx-_?G3+!-uU|V97|Fn5+0t>#f$c2d}P258!oQFrdvWio)|%LS0b_y~tH2wcd` z)6ik5<0&9o&Pl4O%0VE;-ow}-XssPSIizpZFh5&IPVme>bB#$O3-lDsr8aK<7dR#! zV%f*v{XA>X#*RMD;Xl4O{_FA2?jOH9K0Z47;hVQ#U!K4Iksts3gQ)Ur{MXM%Kd{$7 zMiMzb{_8G|nR{BIITv~+%NZV=7ZBz&UK1b<&`Q+p=U#sJ^#{s7Xn_l+snf0mx&D(P z&*smc!A|`8)con${MjGo4i~!ar)GiwJ|`9AtaTA5lM&(P%oL`6>iLsQ;FJ@k=6wOn7MFo zqk{tB0h*9e3G;LW_yu<~FEVyQw3H;1Vuk3crve>R=1k(8>OS--;zHA*7l`$OUD<|u z!|{9}r2fKf=Ril@hej#Akvt`p0%N2gl|WxXT0?McNY0g!+K23p7mgX#)2T3`b&<*B znxz(96I^U{3+Lax9Nzm@HNqSMz{E0&!_48!nF-_yE?Dic=b=4X&)G@6t=AM2wTl#p z!a<wlmpa!%eEy*JA?$O5d`J{+68U_`KlDMiaoUEw3A8#}~h zm0?~&ZB>D6W~Iw2(Zj+xC5uvCGeMH9Qks>pD?Pk4n01=Y5Hbxyo-GLBkQn56Q%+OZ zG2hgegPR>Q({!m>X+~nEe8GA%uL#D)*Ji2@R(?G}Ym!aQ5Ih@5ZH=of>tch4E#|FT z<85QpM30rSE;k`HwQ2w6tOmFx!BtLV!CP3Uh9{Vnh4F(q4YCq#qBfRQ4UmNtP8;0b zb{PKyLmaH@A+CCIbUjfun8>m-kDzhb6ee_-JH~Ww1RtfD(BCz8d#L2@C5D>wvN4cx z<)*h1xP{vj9iWIQtxPv~mi8Vptj$UqR&lBmQi4lkfk8WmXctaNTW{Fhn8U-4Euc&Y zO$;}AE7S0++tRT2X=BVsq*eu5CB6`(N=ObgJa|XT3)l4K{xW7!Po z!e#X%oM!lps=gm~pt;9zAQT11u1)i0le^|iw0OW`O7){v{RR1=C8Ag{RnS7Mjo<-1 zfE+3Nhq+yZ`SqV*h1#zVWsAAziyevRsPk+b$9RhIspWCB#8X4xsJIkW`ch^YD`BE+ z;&IHvul~){f4bS?NE=PS-0=^60Xt~>3N+45jXZ0 z0;-Ew@DeGw^6cAjJ87#&jhoA#Mem&+d%yq0UjzhfcYFXRb^B>YFP)zu#)UOWBIJX+ zxm`e66kTFreQOaLr`K%yqphXAC`JysYd%o&`mPofSKgcXY(oLkq&7+nb9pm}zo4z?1{ zKD*I{0&ALQ2CE$L+5p1Ab|-^^K0adtD7K##|GU4;W0A0j;Krw=P#Ijh!oFs)T zs?5Rg)^PQD4IMqfJm@;8EdFwb>INVq+bA%r!2=g*dP-uh_PY6oagiedg%wISZg8;m zC>Ie;Ady;n@J#}q3)cxQJcNf<&vK2xuBs$qtTwed?$T!1k=w6{l~>XQa%5R!wz5kq zK?Y1Wf`)e`X$z0LF!DBb53!yqkzo87kJ=J;oI3(L*bXe>J6~cVag-`aT&Y^OF-{QR zs3TS0mS*~@H-gK57BsoOh47n1<^|O@K>h?}DPQmrhle^U727>rY)gLLaLJK3H#-tx zj4t6MQI`?l8a|jk$2aw^w=>y z%|D}rRa}tn(R)%*L})BNm*^{n{ELz?!gHf8QK3o7m-hNowJUa1XLlCZXrAA%0S~g- z255xLGI<+w8uYDfKyk2;f?T>hnv*S|JDRFt)@F%4K6_$L>@CYO^S=+{wrburZoU$- zWRN)S40#dJ$M_tjr7Q|-iqDU?#|=Rz<&8~Hvbh713kJSBMDG-Ym8C54YTWK-oB-@; zRl`e&37N_naJ^ zX2sfT*V8;>C_@+Ja%(_0IqH3VsH|_>{GKGN(2<69;%}chxnQVbw1-PlnN$Gm8;HU1AH#=JrYX-< zbLU-l-?EV;hwrtqMsR}D>D_JrmR28k2uJ%6>ks{~XFsa5ej(Bs)8#-sTaph9683Sq zU*8o}7+wA=%Cr8cdjE0TqH%gaM!0JE)JOT%Y9`&vCdP6HC9|P^dQF9q%x}2tDOTKm zYz%B=(`yh1dBMn;2_FJ7)<92O3G3-B{9B!c4|dhJPlLT_AATBY z?GJBU+gh~pD08kI&<<@CVbHB&(0Ha}rSoMSA{&No&e&`_H&Q>I}VX&`-w&988!GV zG@l;Co>3$N=su$me72u^O?DXJXZwjp{24VdZFHbs1fWqgBhZ5S>n8P+j6e_CPd9ha zghr7aMjnbq7uumZhUeSy-Dh8p38r>Wyg3b@A+)01cLR_(+s{bFZR~i1Xh+-6Lv+2- zD>xP%DO$H{VJHRAk|K!)FqJydlOjpSVJy{XO3_M6H0Dwy#j((b+s|WZ6b}nIw3f)1 z8AZ3iSPS;{_Z`&>B4+2pqtZBfT(z@(n^+UmOiTjIh$D@(PqhWIQum(FUP4};V~jY3 z6~bTwzw09DW$bfh=Qx6F8@C1`B`(fLTJjrKwh}&MX;VkXc`oXFYg%`Aw_0L}AhnT! zmYu`562lsb^0=@qO^T{4q>Ae^6+5Q~j5Rjb$fed{c4yohHr-OBwEwjq+#);8NJJtw(`E`Gh4EbRwJ~yrVyHW6e8s6L}A*q506sU)HYM1g6B<_q$jo zgud7NGg4BB9(9zY8&+Cpuo}HTN5bazz7UP3(^{5S9^&HpcQ5NohFf?^gW&lDQA8Y| zlC@hisyI`c+|rE$r=w1ULL;v$*4aa{9roX(cVMibqqt>&rkNBg#nV2bHS9)>M6$J& zdW^wg(=RRfcZmBG3nzGberCUD0bg45D3%hHtXWP`M;YV@wSg4j7~~SH-!>`ef|Z7( zi?~UoxM8J+;F6W#KMmf|*n=7h1PtgJ+`o{O4Wb@q0UOcZ;__4?incQqcHD6*u3bo8 zBJ;cwTyLI$>|w4KS*j-~yJ6WA#aA<0CTp%)V(yVAwBR!spa6sKJWpeyRU$aSaA<3L zU0pmYZA--UBDsL@53esS2N%@}1Hja5lwo5EZx+@}Jr_$15gytmH5wBWDb#he*AO>) zg{l@1iQz&Ih?aBmGUOgLd;;NjpjPHfnzQUBRo$Qv`&?FtfS#FUdDy~+ab#MH``Ti9 zy>+E2YlBT)tg>VoyFpx|fr+D{ruD%tbpMqV=XmmfY=$rA``8epZ_6zn=Yi{>ccHRi zN0pI;uGwa=O1FD*T7V{7qdfVigVq2`R=IJg5uqVTnE`79%D}}+rc2Il7@_5Y>yiR6 zCax>oY9W&@ruAubd;`R$q3ZBnS)IdUg*Mb3!cteAakO=)y48V7v2wPmc*>r5TVhzA zjx()OMPW=lV6eh%-RRWHwg7o*;s7SgE%g z1N9?75jN|)mgPPzHrwVoP-_RKHBv~3g%f)WD6!8GMl8w`KGIeY^8UuchCK#k*slN= zHWn(($8rcsMlmp9?F7vL#UMmj7XVm*<}f^16!=%5g_a>T68J0F>0Lmp0O(hM;tt^0 zo&dkbf_(v$SHw2h8_?GXw8!W~>BmWu-hYDhV@FzQ#p0f{vp z4yy%r6`&psg|!QG6_PmuFj!q6SD~pGg23tpxC)+xLHMg4aI2WXhoG;vLJmSoMi};L z2h=JgP6QyYdL*QfyfpxK)dOY~;wwE+S8+gA0phzbS8*U#A@1Vg9E}063N0B@d<^ia z#{jN+1kkFHfK{H{AJjv`U{w!Vw0@wfT^On;<{AR28iR%r4^UO(2aM8@_6JUN@2H?j z&zkhCNzaPyYb4y$<3K%)fO#5?)E*D<6rb1=1@F`bZi=89fOhHtZHgc|1nblZ*c3st z2h! zx^L29G`P%oV42aNGM_b|%!9#X9s^YAvjtQdg?kqZq%`gE-_{=A7g*9Lpd<%@6eHL~ zfh2kGM-eo`07?DOM-k+^!IAo5k0MBZwyYtxG-4DO(mjBX!sg)dAV~2S|9J4DvA{?5 zG(>EYFz8Wi7JD?T#TXbP51uH3G6_N$?LZSnQGFPE(OBrB5wJz^kVWIBk${G<%i-e$vJPeNbZP^b)wB1%T z%^nD%Zdjpcs$K9yosdG&6dwdVG!k~`9*{%h;CIGA4aLF?je;1u2joi3*haw$-Bf5HmQXt19!s^3VUp9lxa1SoES`NTo>M7Uk74JaC>CqQyLODGzmCpe)9;4=oE zCpdcE6Sve0%@aEQ0NgVUmM6IEnrELcB+ui)@!UJ;X4Fj-hType{7zWS8U?-MfOSG_ z>>;o_qb6^470PhEKfMV@p+y_$r5viz1WKS7R-nShW~$VX_Pe470oPJ z%#)0F;Wx{IAU0hZeVX&}E3t;~JY-qvb`rpKvj@M`V3U(NR|DSklckUE}g7%z-mM+V~HQ|BWMQyJ@7NjAR7Zruulm?x>+RPh|ci-^>zdw)XgaJP6P|EokWyj27~yHk8Z| z0i+HPszlv`pe+t^PFnoYb{{A?R~$=84d4<6_*}zHYgA;LIcikgIr%6$&vq10i4825 zOS{i%^h($TFY6=t0XFiPI4|X@WGcwl0KN^RZ;)#69Qd3~X<7m}l;S_xt0qp7<~6{{ ztjMTf^NxOgMR{f}Yv@IU5y9`H(Vmy^y=iK<AT#$#Zsf5#x^U)8rU?y=f5E)$ugc@kwd{;;(VfuvdU}$X1zGwLuIw`mm zpdJgOB*BQBq!gg($x$B#IrZ71EzM-*W^-7gFxlb6yRhGELn?7C7tnrX*>%%~ zE*M`PhT~?%T_+S;`z&CCP)vkgiLf`t;~-?#RvTarz-t4ZSydQuW03`;t)+Jr)SRUy z)tzh&=lr{uA^R5$)`6#i4cQ9#Hz2Tjk)TS+gwt>|9ITaFd@y-i$9`6x9bV$71IpvR_mqrDljtf8?6@2>H2A)1B=ro{oDxyo8 zT&tE@J*dk$D%Q^Z-AijfG>4c&&?O$^N#&eW&V8z!OfJdq8^IcZgc&!lA4~_aKbXow9rL~>$H1Z%_U>*Au!L! z9$2pIB%Uq1X!^3%XxcP0QoU^f?sFJ0NobZ-7(IyXj-6XCiwPktxx#!YmY(#C!pI*1 zs+NLi>*0d!h1~0D~MaAwc=gM@Wz_1;f8Ho-$5#t=!Ht+?38XLAcD1nAN zBDH)PCxN2$7208=MNM-?0z^lOnx@HE9Xwjpbi!nKwW3AMMwko`9Vu$=BkgfNqNbbO zNl5$+3W*-yZRO`$z7~5g*n?Gxl$F^AVsB7l+RFx$0{6}s&t>zG^C>4)3E@gF-7lus zjo&NII$Xzj`D1N}c-BpD&_o$sQ*>=XJXqbgtpJllw3u9IYDA&K;dzbnX|`soGsvas zjPv{ky=^v8-^iSmMq?k$@=d5_;+S-gj4FLz$_2aRG58f*Lv4Ls!)}=Wz6BiDMO5Y) zu@QB4hGN2gm$J;MenHF)GcyfFd^0Ly*^CTr#HLI4o}4y`)(pi~@vi5^D4dpnf*WRx+AGJq7z~)Vy(#0?58tZJi{uF5X5V%t?FS-(f&;sMY#V4ad*Vtmpl}rbWgJI z^4NVeeEnQ*ipb+*_Tf!qGcs4tFzI-e^rA3)(v1nn!_AE_k?pQ|H&x4q zjVhQII?RYiDe(`8d)`-Oee6B3ouUK_%Ke3OC$oE91j$)lGuQpE zm9d=1us2I0f_PmJ5)TTLN2nlFxe__ktt_?^urD_WqvHO_UNemH84kw~<>-+!N zF&$${#Ny#_!=FPlOGeQ|=ly5hOw48LI^t}>Yib&T5UPJ*&IC0b$D4i;lWN$`wVIPl z2o#W+#chUonv+_1xNmV_!C0znDuP$r;hog%1$oDA?KxqQv2z5?W=#UuvVwze4w$NP zzeMJmCD#!aQ3ClQX!fo9VmQ;1^2{#~m2LK0&RElO#qKLmkZtZ0pk}#J%z{PB6IMP6 zdK`#{6q!ho@a)$pHxO~LMv9EZnyNAK9=yMO&pXQ-{@5p!{}P!QgD(vHh&v7dg?1;S zH365+n{@^dMcleN(y_`HtOS6Fq@+dX2Ehg`Q?Bz50cX>&l=mXrSZ2kASM1o;;_X`d z=kqWBFn zGbnHdLilA(9Aq-BX|lE%ocIec!0uD6DOQ|?a-CT504!l8FEU2{QYY`9u;r2^`aeIV zg$dN`SBLh~&yS>?&z^a^W}7C@)glpL?TKEJ_2zM4E(pW&|JoEcnJ>7&&RJuWY+CIl z>VO+W3Y~*)-FK2h@Iwi`>n;RBXgWBPrdgG&{l8aV%)5c~WJxbj4gZ_g;r)n_d`6B<)p~ z+RStNtL?kIU?rD`7&~Nw-#REqSlq|$S-Et4r#{pkD&PQ7KSCJY_UW<5$?Fq4Cx6?x z#Pz`tVm}(j_R$_2yF?8WSjrNzF^^J7hH8wn8=mOnIr(3#l=ik=QO%neEO(u2ThX5? z@{}CGXZ`)mbC#lHe(Z9BG5@!3gA8Bhc0z-mbLeu^&iT$&w`gVzjj&r=!Cs|y|4~x16`V0VJ@-6e5u#+5omI<1W^%Q;C`v|Cqsi(2pS{0t zjvKcU#o_%sp8}=aDoI&&%l0IhT;FW+XlzfSjqTC3G&6f^XP+2NqPr2X0frzY_iXmN z{|e^-Ajl>`uv_xj8Dgt;oECYAmxJ^EQUC=QAvfhRJys49Q2+`8@C>Yd<)|T%qYmPL z5=GaxLv$2^y!7+oJ(1P%04$`Lnx5Ug$622{D#xY7H|%VznBBM2edo>B zg|k@oBiCzM^pXBSg56j|)Q&a|!A0ejk}RQ_hLJ%vx~Mq_+X2dIg-&qM0 z1)MTWTnamcYc)qtz}PT&(JVt#V{s}sl=pfzW9*q@YJ@&cbyj;sF}Z@P}E`T022KF8Gg9APJmukHC*RXs=4^ECA+BY^H*$#cZk#a8cpjJloi z6+5Gg<3-i#tZQ&aK#Zu;*{Z%7a}Rp!bM9MrGrHpD_^^bC&5v%pGEiAFvZiLF6*gw} zkrgz@$Kgiq^JCP@e4I*|<7;FN)oU7E9WySX@*(PCMqpUPBjc-J*2nV~wvAH|j@~rh zvBFDfM97+BgXIR9b;AzSPlr+>6AiO{Q2iotp4qo4whHf)&KY^b2exP14h_;esN+v+IqVca_ z9R-+bzX3+{9kAO$W00DY3nHova(}`tZ$ZOAdlOuBW2jwe!caC3zL|;~Sa{hPaH6N) z+`>z7CDWURirjzhy_NRJiT%gI+1pNg(@F+ddjz8yW`qq&r8hW+RDHOi7*l z+-l;IvQ@!gZ}57-bql2`E*PlW+xRVEiSvwdy)dDG3# zvu@_*f7TX+#sorl6c{^SLCgtS6!bOxkrmQr?A1V94+h{Wk}(Jdpn|;N;w3)cD6(<} zUngJ8ew2cdB&$@(ivDA=XoS^WHR3!)Ndb1)74rqU+8_k|W-+t^QbnGfKRbW=Tx%fc z6QYWpb)1a;_R~4}M=wOZlBeV!@ZV?Vzh97lkbe+zMXredU;Q8Q4>Hr6Y?c<<&8KHg zLYh#46i^dxgX6$v@smJu5*cte=p{e`(i^LWSyQovPxjLTOA0Q;_MUDB z34)bdR{YMj56laeu+&x8#CyzbBp80IpMvpnPA;MDF|DhW0_$-ewz}i>Q^4&f#@*5{ zVVPcSeX&QLPveVcd;2st?gmGSrS5spgHAiBlqD#=>q{-+_#7(j9kf-SZ+@S*gg2A6#3z(NiTjl0#B7wXiQP@I`cc!f7@P*oE7>Iv08VE z655_rxv@X3eSqX^=uNT$CtI#2-a^-NU#0G0c(RZwyI3x{ z;N|v8W;}_D)PiDi&G$SWy_Wanp5e9w>h-Sw3OwrmB+m8!aJU70%CR~lwOW=lB$6vQ z=5KNgaij%%PG;oKQpn;tO1%QgYdcofJB^@a$}j5PV$2HS^1(Xsd|&9!wvlCKzQZ5= zp`Ry$FT*ifpWOhD;b=f@23_gD!nOqjYK;N&)_s~~?2cvbBG`Tw9+HPHvBt1Vy-dZW zKokpIvt>qCy?15M!l8He20}Q-UIPEE&=r7B^wR+h9|@v+Ct8P)$3u6-98KM2RhqAz zK~NgrS&2eTm7NDD#?Oo zR3!SMw4-bSormD#2oYw5`3ZztOBsC9i$Uc=mHI0o#g2Lly~vb2)8xhi-ZjQy>nQkl zzg7dvieZY3)wy8K^JUJx#rQ+j=GugHPS|_}F7*~_9W{GStfeXi-LRv6g%NgDV;6sQNy$^;oh9mc8b4FU6JUYX#{%ESPd0!0(qm9{ImR**ceed&*eJakF zzN@x1ITys5QFBlGy&jtGW9z6?G9`|Wo%PD3QnunFwK&w!3wt4-Sn-#XEXz!0rd7*9 z{VZjze{+D`w9NINM+ZP3m~mj*{8y-e2mCBlc74U~py+svfe($LYnJ7tU}+U-&jFJJ zZhzrh10dhP6@5Hv9OMzU{^-Bt4W`YPK;N*;+mf(5R%}Tw73PitiKj~OmDq%Q`%4>_ z1{x%+&MI3+M%y|9D~@cC=ZCZhct-rN-D*hjCtsXDJ%75R;P(Rc75^*vzuGEq|9iRY zu#*HL8_b#-Hzm?=AM`0a13UT`r?p+5d~yEl{OO5a^RrXEde1WdZ1(h&1nNgNAexmi z)v(Q93R9-cFLzpBa5lCP@5R5->oe=o+oi?4Zg8rX-N6&y+$ z!Gh2B2B6)n8B#XY+&zsl?^i$ae2=}+ukEt|Or*Kwq9@n0ndxaB#XuqBkw-*;%$jSL zJdqdjj;FmU4Hw3;M}Dtm;6FV7z21e+L{%{IzGh-ouAh^q`z&zHXv+H1NXA4g4hpL6 zJWmsP1+j=miG!O&m7UaZ+W=tbt9Mr~^u77>&CQi9T+xGE1MzlTD3(-3%-z3Sl9Ve@ z?=4pYVgTlzN->C^)37v!6fEhl9fxyTNSU!tp-pKyh>O+N1qCQQN7Y}jHKt5!L~+)} z@~@dqN-X4pE_lYvEup4)2Oxc2Mibz}VfW#5Tkpx~z4!0b z`}f-W_s09T@_$#Se?0Bpx}jXZO!WyVI~#TL`kENAq?B+8!J4W}`60uUyO%#m-x38xegb%i=BZzk{wjr*?$S`xS$3vyR9NMKQ z5Uqt-1+=j=ECT8D zY+@x%R5?8<5qy-GhK#~yDxnJtJUG+RDEe5#mCPrBa%dfsKshwKpB-hN1j`VgXr%!WY1WHzvdy14vEtZib&)BYP8J4P0>U>b4(mFP-Z+ZS5*dHU;^%8v?HsjP6 zHGvilNRly_bIZ1eZ8X-3_$2oEmTj*GxY+sr^aP&_#II6gPX;~t5>?ZMk3sqHXN z`;__FSY8&vsUr6+jzf*-PQZJHu_1;t#c`!!juhrbQJiQWE`-jXa0&j#{(fu1OwfM> z+iOE)ACQIx*W16DGZ!hb^QQnrQ!;%Z{^-acIh3!j?K4>GKV^9)x3-9kD*;rg&dUV> z=lUWS8_b(6C0Q|1*PHd=UzfDh*X^2>H!6TH>|Srcl=yCIiLtZQ4B;%*ctqDoa8 zW((n!OqHa{Rar3F)%WDCbLsSs@(lR0EsPCK6m3Rt{Zi)^c$2xO%7#;=9VrC?Y{5eZ%mlIAl# z(fzbu)IJsO;d5H{ii$1=Aq^*E584P@wDhuvOuxN>#@Eo}2MVAXVFIp~0ZERZ_jY6* zq;y%((PkMZY%Je@y%viCNJWU!V%$rUQ6wjM4cRv@!D zzV|E&M$>OZ7MJw&##G$ZFU@Rhd|6cN41c;Wc*!Elrk^ZnrkIw0D&ZKyX|oPKJQ7}3 zNXSvuV()utE+8ObeJvdd9{0riy@tI}5`jE#JXoj}z znh)eLYwlgAw-%iAW@2yYAn!>S(?h}vMTecY{;IV3z=rQ_H%WM zIwSXVK-`st%-4ry8ef9xnyZm%4iaga1(WsJrLF<5z)l{vHhQ;qX1X#dg^A=;Y(3^_ zE_*IJ<4y;Q?cb|c8C4}uzRF~Bt9K(Zt*-0M#!3ab`2L!_Z^l|S6yn9X!K7c(d&O|M3%y+ZXVzzbh%rGeRe$P5 z*LLIcHZK`;FzCzy_uATVBAP0~PH$yJ?kNWF1Erah8=gNWuY{_wx=-2q%%FV&^3z-v z!=l#OZNp8mvb{veD@=GBa6}!k59P0q&dAY+rPBWqk=`c*099sfZ;j3dEKL zGcN>T@})1HJPJ|-zmH}vEj*`^+-td+-D!>py&+xEL+^Ao^A19UTobwBURN-cSSMBF=;_g%TwD3*%wM;AE(mcc3pWu+(tdJ8PGkWy zN~j(Av9Yo0ec#3rx{a1heVapArU$T*+CBbhHcnWR3>p^DT{;|Gc-OD+#SWr z(&XMX?Ia(Oz>{dBEJDA08NW~V4eTGy|6Vb2`O*@2uZ4fg)opO2EYsCHr8)7*#fJVU z1$p(=wVBrGPqb-4$}?ofp1C~@%H07wHvO?Gn3hYT1ReKuTe#7`lb5ulDB}PH3gr+x zhN#7t1vJlrS3Du*0C^$-vcGyG(-A_2_mU;qf7Ua6fxnZDhDP2>!NS1K6z(=)M&KG<0PIP8z?!^ ze>(@;>ggP4=AhiOX2p)cWMAwRiVl=thIz52xWNa$eRp{fuMP#x?DA^;bSJNdKU}lo zjwftDVK6qT$79#~BUAdhB9je&9mUb>S1dnt{q$`Vj^9x(xM;)zWAGyvh86*Aea4|u zS&<|HD9yO_**EO+kRZa(Byh1Jr!;(l>kaOI#LjWwOPsKLy^O1at{?sSx@j?e!SXfv z`r1j@o)M}TQ7w67j3lJO-AbQ6y1{U`+wbt2ta&9Qw$pYA(*7UYfhD>1SzlU{fN?C@rmx7uoE^5qgRhHpimjz>IzNP8> z<4X~>;iR$D0hwY@T?2>@%?8NFLCP`QCLD0v-s}3E=5Aj%ejm~va}&9H%eDmi6fR|< zakFUR5-Stfg!4+VVve8wR5o{r(ZOWjv7S0C{agEvExUHs{OtU}VS@7cXk@oL;%cKdRe#TB`1M(y*8BhIIF6e}xv+tVJ(^L@dg4Zz=7uJVz+zm!2cH}H2-4ptsrN_m z%>%YGPR83`4yoxfMD#1$nS4+bLu%!xGv&0P8&IzcrZ?0~oS0Y+2>rmk zk|twKSnAbCvFTcQA^!#_5OyRBW8JXIiZVjCZCK*D3T!(WErf(x77*<90Z{b{A(pO{ zw3l(Eu@m}Q?m6Q+Qd?ubLR_lE4rJ_~7dD_Fy@rWW+w$Fu1czh>)0)v1fr>aWcPNdZ zJjKe&9210L>Nzu|u%~Sh=7tuxT1PY#c@9}8$`zd`SVl{Jhu)ie>+w8~$rN00-AG9jWW`Hes zOze1Lxbio9a%CJHyt%}zwT$`9*k`yAU>}qv<9GzdjAIOY#=PU0eQgpsbK!k~%F;SW z6^wv5h9eJ$IpcU>zKml}?u;XrN8^a&)HsGbd^j3+%)0q-`G%w-Wn8*Vi`bD;k`0Lc zmWa&B8$*=wa?yp7SydwN7-V?Ur0KQP$1><4hQ!)E22?x(Y~8yDiSiK1H_MA9bPrXz zXa;MIa19S?+A*%-A(O|aYwmb(Q%BTE$CNc`?3sPRCOzz5@()D4_{v)`mc9r?8gH&i z6pyncC{9jZxP&_!Ky)ceRxD}4PJIgl%hznfia@h!8w_c4-quv4rbTSR3LqrxL&*w# z7B8+ZNm}qbRw#1vRVa1cbQJ3K{70E5;7jp=%J8@)&BD8K^|W!7xWpk9)md2P8s8v6 z6G-7!R)z6-+3GKeEVRzumk3)AsQR(eDoEiq5XQD#2Z!nI7aCo=T>7JtgKIKnS8Q(` zlx@JpU3U-4sWR3?aFq{rm%-s38>(0!9l5FxjkK%S)IdpSo@YVld(>VFnX;>Dk#V)Y z9&PmRdKrVSoXS4b6qh(}Lx=5OF`+ofC9R4(`WS}xr+r~0bAgy zWl}-MMWDycCVgmH7(SUG>P93;z&!UAYg2hI?rD);T=nLUHAS-ed0!~CKNI~sT|irW z%wY&Pymq}~KcyF}%fM#JV7EP4L6qq9a)r!yL=AAUJ)CN2DGNfKoNAL(?elP|U7}M> zbJC0}D5}$-H?|B_*h)9nZ}GCGbSU1o)4n`Od`Oa!=F--u*eX+;OVc^|fjl4pYe0eN zSo3k6LW0e_(K+aOAaglj*24k94i!QoXH;j{AzYoI7O(LYB+Hy8ELe@J9*(g_n3v-> zia>QIE1t$Y-LAk?R~0Qy7yJ!{e}1j^t*63P*Wik1VXkCY%!CyiMdb3V5tny#_jEva zw_}7`idcjR6~Aka-BNhYRIZVwE?gav3SW=f0Sng}clHsM8)u*nA2vjbm|Hk3e6$6O7^qPqNK?#+02|{z^qhk+&Z7wT zp(j~HIcH9MJA#okcN~I0rs_2c)jiIyu1~=Sg10&03F2O$Z?Ge+LzVDsr$#*uX;HLh*skVxg+1!uF?Ked>>$}mC zd7p+$p;<(^Jkpe60C`^7yJ{VQ8in*1QThY3Xr{sjZ<^;BPoN+^XsU3`4sC9rrtb&7 zsWzli8p)+F&jg{G8{;N;Z)^EOaTG29U{ik|k_oQePS&O%$SAeyrK&*LFe^a$HwaNK z!M*RqP30=0?_5T~v-sO+3s}2DsDueb`zY7vlazJ#lyHWbn~ivj;%-Oo@g{roDnolg zalOk~Yzeh-T#ou%NAZm_))=;zY!g$tg-2D3$7vwOKqAOY|37Q&u^zq!TVlXU|EsqnT@>D$oYdH-zs#p_Pb}6&j~$kWQQtLUj<_+=Er2rfiEhd z&dB1_h6wfpmP;bEg5bJ0pcScLdV`^kO(8c#37V_5gkE=>q1tme))~Mx08%2yStrmjh3uPx)pRdbr<966qm;P1M?u1T)EQA z=x=(FQNAH3iZSxaqBQ>HIxH@uHL_^Nib>Z#N?B%_t?F*RhGTb)8WFrg;qGT-iHdHpi(W^gHXJeX`(2GcH*AwR{JA7;Pz zx*h!Cz2-QDejuVnhvqlC-e1Y|I%mn;^yhhxiZNlLECwoUUCGqzq>OVe*xzjQ z1RR;O+a|PRVlZIYX2S|unx?u-a8KzyIRR@S7C$&6rIbK-fEET}UI@ZIa0Qi6aBtyg zCiW>y>;LjEp-YKjKs z(nPWAFRdv?cID~d;^AzlFjul2%MELnBB@0oIM{)LHg7=!%C;tI2(1OiuOnE(l%mD< z40@XJ(#Aw%L~VGk`UEyqVE3?sB~qxes1i6SN*0B@Wx}N zhAwMZqs)B`i6u)(T7fhk)0Q4bv}IOHccXUnrcV4%>L^>-i!NDn*m+)O|w_5RCRDB@-s~sjr3wHH^A)R7zUwIW31>_4Db+@G<->O-=W2i zanWHvxHmG5^XIi%x79D#$rqU1qrkj{4@9{gHUOn;_qZi%dbhu&xt3mMuG0)mX$GI0 zJ&kdRL_`$YG1Ai(ldo!rrcu|?ZlPA?ANQ^y^(Jn^MFvH*r?P`_O@f}2Uo`|ABMdu@ z9usm-tcpK32V!eSH6M?0Sd-QVK{D&sQcdDeoHNl%{#p_*z@% zI5cQfF@cUN==wTy<{zi32zDO@)5>R5VKqq`mjGrj8?jfI>ZU%&Xq(&bc~JZfJj=12a9{UMU5`5F5zXX7p~xc zSZ*KfB(`J0G&!g%Cc#Gg-xwXDhogCi{{nqGjN{F%fnlx$UA##%@wW=8i6}tk`Xq+zn zBseGQg8e~%?Tqk#3#~sW5mL-dvc7ZP7`fX<5Ti|jia5#62yHN1ANBHue$8uP&ERz+FxJWfW0JaE*3{@owi@Zw4Z0L7Fu33QDo((ON zHB&^%D0C-;3surAt9x}D1dt$c8eftxh{lm{FZkyMLFV42$@7hT)Vv!E!OYJy+OxQNCDO>XaLkt!%~KueAt+s;sG z^93w%d>nU0>tp7kEob_RKB@l}cTL+e>Zy}`6>~kn>P?6PT9tA`OP+uQkS{&&+QxOM zu(1?zR}@4M5SGkgnLF7ntw!Q~dm}yU!lUu_AnughK4B?DBafL0-(Uy{c+v|Nc{w*) zsiV*acWG-vIG3q6;hlqwlM;SKz6T$=2MNuqStjolDetATdZF|ecU((Djnr979WZdU zc&U#v8R{eMAOg5Eeb?_}C}~_jZLc1} zK#Du2AQ;Fd{e;>x_+!NcD`*LhPgB8@FA3e5YU3*Gy%9bdpp)4i=}1cL9lL=5=0uGF z_VPv^7Zip%CEuDB@Q%N@yzCzPK%5YSHUzW}py$Sz2*+9vTmPUbg-qY<|u+r3{vOa-#_Pc*7=s*#2e;BpQyLgr=*os+u%(|4w zQtNmPc>h?L2m$?f$T%KF#N5u#$u+Y*bNod&gTSI;Yl@w+lJZQ|y!57U8A*VudDZZvWi}eUu zRH>ap(5kK(pD@x3=?JTN z$w9&H%l;VYtROBGf4Uz(-}~{-mU(F15ai79hwJhRcW+3eNUr_f7@1sdcv&(V=Im|; z#HXZ@_4WK=XaYD;Fs_+>aJ@D?1pqYddmhNbXb&xF3&J-!wnBirJu^9Y)^RX;2NHxxohV9nJU{Eb>=~%I>8&BX4D?|MLnmNJiqN9xo+RZ)FMZf0RSQ(p@^|8|946Go|HLvu}a{#y$daDo13gdWQa zH76HDak0vn55t=Eyw630)ict1(%&n7M>B@`j8wRuZ*sXKf5uY_Z1So_0+-u&bj!9! zXKlI^4e{tw9M$zd8#Ktta1hgQ1b>SxUrbDqLJK@RD|EuoOGTe`^9(9;T!Bj3WM7MJjMo2Jg zXlZT3_DYghMJ|=;`_#TOag>w0ipDlQ6tkAy1%VV&leQcM@uFFy%{Z8#@Rac_%kGVc zf^I`eQEhX*Zg8rYSbltraW}@M9*e13l`8!huu?V`0_pI>ovrOq44ZI!{n8yT8COik zA5f2LAuD0<-d6cKd`I=_ZA}3;J#!oC#K5>oF7-Z{iUC$L@cfrcRup)4i)}aMiuz0{ z8J;wJ=?u@VJBD*icASaOrR0j?wN)i8*%|gjTxPayN;DJN$doTl1mU&k3_ZhQ5~d!6 zUxYadtli#+bE4vZYl4?VpF}&Fu{(}(e;f@&Qa%zV9qO=TM@L^TsBUiypfyTuMih{- zgZIiS-vK?9OY(JG8%Yl$-yqS%qybqHE|RQDF@6Y1sHRmhbbgO5-Afv&U$ruz&3AF3 zMX?0Fi||5Xg~Qn-uFk@!)kjL+F-rCfPABj{#J2_?S@ssOXf|X`n&!u^`J!Ot#hMDi zGOrLby2g6vOv{ocYvcu#Xu6fL@>r3zR3+KaWX**ukttj`?ScG>T=t48F=p$Lt!a-< zZ(4(%b#qs|_I1nJH$!D!pq*O#{krR(Ap@{AsLld&f@i}>+utmgTyC7tk_(S7GxI_a zE^kOTbJHwyE$>Mx$vuNWBrT^xW`T<7nB%P;lJXL{$qTvRHcH;&#K8r4vntpUsd>U# zCG_#uoo2p%K}uSzSgGLmzQ{+SgIt{HP z!n8Z86KvRcd4JE-5t{*zH#fnmzIqdg-UY%_CQ5y|i?Ez_0?pRv5FysZ&_sSaj6)Ul57!-Yw<(eqIsj`v^CM%U~gXr`z4`GhMs)K>8UA1K8qzAVaz3VX&_>ccJlfoQ7!=+Of@kvoAl~ zOePeE-eIwARZ9t-yj>|yAjalqsb--IG*I#-5#hka=Ik2Tyw*O~xq$H1-(U@QDj*#n z0h~c1GX7h2bS1~tgePlO8f+5~dFNwt>YAAG)?+?4pK7#~P2REMj-Rh<5f-JwV-xJ8ly>Nl^*We`ryn z87-+|<&Zh9DWBEvXzJnL5%VYTKe3e10g_>A&yv|6jF=}HP?oZA8)MrIh3T&iuZ3^x zJuX->Q2jUx`^}8L{r2+Z*lp79SV0i}-Q21Oj~}>s_}?pE1%nd*ZgUK*!&W(Xi&hD% zO{Ooox{YhYwewo@YW?ZOD`I|))LgplghPhHez3Bx7=K_-n^nSUS4zs&?IEqb_BTDY z^#$8`qHYZn#J8=i$h|2rZuFEJo>9%;gC zn<;~^9~v8iAB(b=wr!$ATE^qpOLJP3pl-z@r+dYcLhlE0D0W^hZ9-4e56+c)D6C|= z8giNo`55@M>&rsUQ+9W*Qu_1^rVx{@+gvAAVp%{~`4iOa%G z1cKV09FY@Qz>E?qNXBeoF+*Xts1*$8Y#)p4kq0+_=jkZXOP9ld4LLXDR}9+T!&@9B$L8~k7%gITD{^B z?!$!ni2ifqD*>R4TwzmJH}zgaLzFgUTcE}(39bjNElh;Yr}}qfNrQ%BA)TFu$O4%-@Pw{gTQOp7lwheU2R*o|iGN}k0P zxO9}`ur&-**1@A6rmolpXxV#ZTlCdF-qKLr&n6{39J&B%4bZDmouf^QEoJLL8J2eL zCUySq)y2y=NbEoH$|F?!186Q+iQ;ht7ana;TR|?bUtC`{{UUqD zPe%k$2NmQ#zepc5h1EQateUK8eo>X{ms~*+#&`i`=S?pM13Lv9Zv^3bql1WA!+7!M ziz`Aay?do;(|Dj{P_OYCbDT%7(7boR`YeRUSlz>h&o}&71D1DM?%~G@!!$D+?Z*oH zuRcO3{(+nZZuCEk?}`gva$3q_l;5E7mNTJLs+1eNVPMAOgLNz3f-&!MxeKt)3k4kr z_WV+4tz@Whb2I+=j0kpb=EUv$jiKttgQ~uwaZO6peWPOOWopOXM)q1OFYqTBGQjN5g_pABMg?h@#K`-Z#~mW_G=Q z^EY{4LqX``H7}eWs$$Zbz%PtRLNDK5|Mm67SFc{fzF~Arz#>_$ja0r*+++N$?Ym!t z$0lN(cB1936XfmJ*9}kIjz_NFg3|WX9jZEsW$z9=lD*TbKiyDL^`<~`xMKd8wzf76%8{MeXRY;8Kjqz&yt1t<0kD4-%h3lYdS4nNl&e*V6{VW~1+JCJV6z2{x@sfYI2yX~x@q;}_ruj2 zYL}UU@+$`sPk=BkI@G<=PCoA0t|5hFcPj~1mg}1EH9t%FF(~uFdrcmVZacF5{a}`8 za8o+r`sG zln%I~Kc}?%hcxLQdR6muL?WAh*zWn|OCw?0Of+SXj6`4=*~PR+=jiF>%e^nAEXsRX zj49I5e5DmZWB%4oBJI~kKCQcV4R^2AZ?EqTh9zRBw2r)jVLyoTyS1+1r*-xA=aK_e z864XTu4b>Q3dlA4TYl3~8$*~J8rNl7!?+LD#`jD@AiCr_TB?eWQ%Uw%oT zL0HNXzG(v?_|$KnKAn^Qckw0^(oamgT!j^@a7J(-K45i5{{8x!w{@LV^Vc9?Q;A;V zh}+INdFh4U`L;eZ3A+RV;LW)BoF2kvauvZ`GChJzGqZ|(b+de zrOK}=k!BHvV;ieIda(lmV|z_%hHYmZRexBY(fc`6#w}}j@9UPfyB~{yYE0+)K2bqv zRmvHZp|`=-bMk{AP*?d3>K?}wGB*;`?tHw{_iHJ~)Vlpabii}7*VoTDiWau3R_s&- z-LNxFZEb<$X0<@YSArF2kBMsB zTr+a@>J7wzgGj7Jv62Z)Zj7r5t`V>ngXe_2QnG}Kej2hlAms{!Lomr~k;x=nP5?(D zH>RhENC~_|){LgM`?MB|@|>k}LSE?0xa0{fqna7&w^No@TCO%ZqH9KEkuofgza)tV z(D0zh>jj*E>lrS1S57 zh_`#W)R?Ar&@cxXb{qK#`bVfD40vuNL~gI1i&JCeTx)P;MYC)>V;?km=65WHS=fOo zt)?FWE6)(A^MaLFx914gAtYpE9KoZJprLU=RbthHKEUjb6b&`zFvp7Nu0^L(^^o{!wo^D%mQ z#&`7`1Jd;We1N{5d)$lBFC5n0GcIl%N2ktfk#H}o89~#LsZwpU${y9!Q)43`YpiIf z_HoBH^14e~ZhZV6YOY;W?2M#R9GAwtlgku~KUI0WitmN+Yi7E87$@pfqpvuQrOxz@ z@{BGrtLHdPzv_HNz=Eo*pjGu2Q zjj85GR>&@+&lpr2^1aq$D4e(+4Lo}Xn#Y}SA3-Z7uWDBzWpvwZ2?kPh{w-jok8-E# z!}?ULah~~l{xHU~8&fC+tDG!Np?%YkvSZ?r&~3VBCF&-n4Lk3{pI5w48Vt~Fe?5u^ zqJ#Vm%}2VF+2SOLJDs6Tw{yk#5paOa+t@rJLjngAklV&O_gdQ|wU(N`l$y$qG zGZ1njaY_rERE5!f+FeB~Ke21AQ^)o7Frglm@V+r-KcTM{r0^?diYrrV*YlFp*F1@( zSR5Pk8i*DX++m-^7ImnHVr7-%bfLt6UM*7!8#0p(EpA!rB&*m{y3(8H%oBxWdJu(2 zw~Ydu@rK;v`TQ99W=B!ZC>Wz`3ymxMhUN+_B3c##ORR+Au;n`5A%}xT^wILiKKAF4 zw&y;(vstIPO=B!MdnxLh$lk!6(>{h6l}6Jc7qv3VAk-5-g$Zz`BfvPLRycsl1xby9 z85K}93Ze_X8VIo3!;;(+jO7JfjRNWgXI(CVm~Ye2+G))r*VO-C==u*7&KY+fDDAjVqU$5`7V z)gc^vS`uTe>wB632a8BVuI?k3n!qu(;6x@`r)YJfcDd@CS+=wMUb>zq8!W zSXI#)6F+Ml(w3J=pMx`wP$6WA!7yiNiXY`jBrc5D&&w{n?6Xm_(N>*$Xk<0PHAAZK#(cRO?Xv=!+9U!r^OF3w3ue<`% zXtM+rI~%L~5U(15v!H2We}`xHk+x;O=nuD*qXp%W;T(tiqH!E%k?F9+SnJBDQwmvP z&aBuoR=EmUVyty#9LxtuZQFqvmFhMM&T16IVM~l}T^R*BV2O#&m)nIIl|MJy zA{Dm8kR-~jT#uiZCB_qPEZ1yUWyTvi{0eW@G*j2!ji_tU9>$sDj54xlG}t;mp3jfR zoQ~^VVAQs%T+V(+m`G%5bAXn-7&%ETq}Mh8-=YO%(%E{(-xyrqm<8^|T5U%5*9QyC zwV3Rx{zp{VY7f_v#K_Ba#(2gz++>4ka>~?5Kf?(Em?wF4W`NB%Y$J>9nFFZ*IFhl| zvoX*h+K41oQ7}25&887G1Y7KuFm9qW=cf0}W6&w~ zA?F1k`!!3XNcS;Z?ZBu-CUA#En^@ASaH;BdG~+2Rw=NY&T1w2eq>v>oqcLcAi9S}@PwJ0ELPU|REVj^| zMVje&`@p@65N}$NS!!E*{_cs^e-m@tHIIR2W2GcneB6NMPD_7QjZZxLHCfB5Q1h4L zd`hJxPvK28DOvrxBpIWsBu}4^4Hs1zTcW^IS$_NL5!WzYS=qCwfv=#zznzv6UAdKj z!mRNvmmH7M*H#Sc8jmU>w#?mUl4c+I;IJq^2pBnz^svXbYS+nwMd-dSLVOZwgs>Y3gt zc9y}Gt~upl;(Oxez(fgs1|Wi-<+Pw~`xCj@P<4t<7($Z)E@XP6X}(y{tlkAL_T^CI zESaDp=sRt8(S^r}ycgK2jgb|7I;%WO-hqgZw5PE~3|!$|$`5dBeT`Y-rh!eV4m4fD`N+yz@V?JUC{r*|t% z0bk?y#mb>tu6WEYc?H%C-RHHv`_gXVyW6U_v%x6)KsR~D)cp2$U~4mX>bzK_=kH!!ynOR&zDW;hGC8t| z)^WIlU8A{knAhp8*vg9BQ>?RSI^W&!{5g3gR0S<|XxaMApqvJzB(4u0d1W>?EKz=h zZg_{bmMs=xJsVA^Yk5ynN$w%!;7;pq;MG^wn&KVivXJ4ePUI#p zFZ`~Bjdx$Y{8kOwjXbgXoR!IW!t(XGfxbU6df@HZNWn{4#J3|lwr?O8Ok|dsUNLe> zUa)-aK#uNu{lM#w2sw#VkYk)kxs*YdM5|@9lv(py!7q5LJj5v$0~Pf?5j1+>mhA0@ zgJEJXx*Pm@{ntO*HVyY#PYPBJq=2+rt(#XsKldDsv?|w3lsrMpt{M(F^hp=FT%uNy zvJ`sX6g*{$IQmp0VGyL?*~i%3Sockg19zP3_G3F`$GT&q9Id11+@K+DKj-2oeeoC+ z#s-0eUv>y2O*ecXKl+m(k;5t($W^_6wM=zijZ7spW0&7VX=Ucwrebz&e)Re?52zu1 zJ;MLukcSAF_B#ru{@_iG$isSUTYM(|De3p10cvtD|XJFr_R7HCPf{;>Fb^ z`4fUVyia){OB6p}$_?egK!tV>-NuV{Q9gr>S2hTaj0 zKA+am<3*-gV)E>AHpKuVcl05}94{~(7s)^r6N-Zq(2Hd;Equ7D^8-+<3R`Ah%N4hc zz%=^c#!?=vM?1+upS>KURQ5{HMHcq7HWAvrLcY%EiZJ*e=P6hEzvTM*HF#1gVRV7u zE`V>r&|jBPibUb?yNZNvRO4^&9l}Lw8-jZE_^O~?3&Y_6F(Y%6xIDQZFP zxQ8Rjfm7Lw0;#)??rk!HWAEP`!|s0~MFWWfmWfvQv;*;K=R z$W|4qL$jy$%$ADHj*1H~9P}=9WT0OLGtg~r`M`cg<(-er#qP~EAI>8W!}W8_A2fS> zM9#RM4IYFUlPSKN13q|Mefi-7ncrdh*USokn64!I_9BUJw2jQ@j?C1KPwI%u%6_O^ z>=9wNae3F#+14W}Q3i9Yd-A3uz&gH`#MxBvYF!?0dWj3j%`2p zz#JLJL0Qb>a+k*>0mLN$L}w?DDW)0BN#4yfj)2&ai9DEH90ArD0!HN&M-#fZ+=Ah( z;fTzG2TLZ1%QPLCUs`95hNYYF?GMHn(?=OHj_Y?i%`>)c)S!4_;xX%z>ZQ~)@R7A+ z5{IY5HU#{t>=iY->~!B;x16Z%XDetMA8Ef6b@H7)gzxkrqe*`{FJAid^x~;TZB1*8 z1L6T((bSkJ=r>SdAUQ?UlXYp7`M#cZMhj3ffnYy;(zOR;xlq4O+OX!D;%8T%WsMpGhZQ zFtXuAAq(Tp@s|;V36UpSPhph>YkJ3J zRUqV2k|Xn{Bh4$fGF;xQSo_Cw{_OOKv2L;r#-jBeBzA%Hc==ehr zX1`77o@OuPMw8?2c5IE~IC^1YP!#6z;agTN2W4$W*NmnllPhxa{l)9kj;SJhycRUi zw{!A?Am3lSZouQp1cIfW2OWfft3qyAxn@-t+x)}7k!SN~Pv%cF%y-uMUB5!^p3R>= zouk#AF614X?~=KY(s$1m13UB*tXZlNChN)*fuL2-=AJD`UzMgXr_#C(9z{12c24Qf zH#b)#rv=@xk`?F%2u)#l8ZZ?`Qx;0HYs}@f>3(V6!i#K->oiS~1;IoMrgAA%sMt=N zvT2;T*DR?DUT%>Xj&SLL9Eh!6HJq6RG&5usmon^`YORWVD-gch8&02`lw*t4|4SXaeZB<{_J)3|8adi^RkEP!>3uc|JfgMXw4ZNzD*Gp zSUufmkzx_1K)~HNe`Xc`Ie9BfQ_95rg)1y_*`^aI~i8zQRBf)#qB--hE*HeBd+Jtt2BTiPF*lG7Et;bqoiyJ&3Z z4W?%Jx{!XIl=}Vd`48O8GE<{wUaNjrp4z%7hYN<5v{*IGcuxth^)u@e??jx47Pc8iTI`|J2M;2Y0cHcVsF)jF7 zav?-Dgh|M!LnNBPUyfuzVY4}j?L!TbKv0g{Lb_$angBQ^vh&YPc6PdJM6Bm$Pje!` zCp=%X;=1BxI91TU4AvH1t4s)mC;$VLNy}t+h=3t%#tPK>;ivOw5K+%IdAU6!f0E97 z(F9J5;VAiwJIXUq40qp6Wm0XJ@bYI@()~z>GQ3m>J4<^WrP^Uv7%SpzWDF2$!eI)A_7dwJA!t;L)DXPtKWUk)EB$xYR zlFKIsIRtc)%O|;flFKKu(?2G;{Ij7sF>*PGSr=8g4hK~>ufsJ+RwTy&3r}2Y3Sx6e zw+_jklDu#ez%V$@)SP?`39#TgEGf^RY)Ac^9QJxf$j>htEAPkq2z*bf;*MR_1>a(~ z@l}@Te=~KrvS;A+)kL+foD}AC zH?Oaselg@Q52DG2i;)XJcku%*;MqY7K-PrP!;A42!531Lh0IzN8c`NO3TWbbfwdop zma~znG}9?shQ%||pzz14^>D~m?UtA_l(Wk3D}-B`vEK?lgJJ8+~vE!)B%n z*#6YK))DTVWR)tly!6g?=4fDq>Ew|^3OfJqhR0G!Rsx0`T zDrKQCO6pwk)r=O&nwKmotAd@=oX-*|?z9vwg_>{DU#grXGb+*<&b-NO`o#=trM`Cz zVq-gQSyQ->CC$8u8TUTPWvVdU8^s^jadpFm-m9(Y5ZskYfaVFCBgGrh{wd2dxz%XA zi>%>Tm{6dLo(k{L5O$=gq$;$)0Z73*a3VCtUir06v~W6pQXOXhc%Lj7*27domK6r;9VoME(D?5V z_*mg4m2NNlRy?>8Oa1=Dgem$RwWx1)UTb6Ve8Wnb(vr?s?wS7Mv|Zoa63IO^{UNE= zcdJ|qRR1}6wlss}OY*+!EARUiOJy-8+mffx&d8#|&%U1{2F+_4@t^-S-`$%a=>L7z zS^`%@R^X{LL6XqiRa5PNGx~)+@+_TkKrD6{z-qni#hL?lGswe zGOkL{!EV{M;|P1cz6%0BxS{PX&pFld>;N-=SCb{H9Z<0)6r+4?wofHeR%FmTAK7IW zlQm`ou#(~F=B&_{j-_*Ais59eWDdA1AL<_K+01U4yVy@Y6H?4dR&2O{)MpUV+R|3X z>NgC$DI-w!dPpj;=XVqN>e?7PYBQzin>#2uXBEz>Clk zx>h+_Q|M)UZ4F{qU%-7EW`(_#8)I1E0gLZM!;dj*_(Xp#hQ_2XZ7P1zK&ZWWW+fg5 zb38cF_t{(h<8CnOCh0*793avGN!cP13e9VJXA9vd4t32R=g#KvI zxASiu=+*sP^Mrmn38!x{@j?zj7-BEi@Q_ueH??&u+t&6iFXss!e@M<9b?*-SxF_Z=U)-7_PQw|?xr?88?%e2F`+@9wl`ura^reG6~K zb1=u_4_JA}|D{?u8=M6U%_!@yfN*?m1X8GyL##c)imC2dJs-Ow`=~xUypr`}@Ot|} z1Ji%h=*Q&0|K}I~GL6R=$(k0WI!}re{@<+5ePn9BrJL*lzVYPAlP6#P<~R7?-{OCt zJo`2L_t_U;eDUkwKKYlYzy9r)zj^YTr(fvjo__h|mrwqMJb8d`4~|NewD^}NPguc| zJn)<<>2ewP`x6}Qzn{tYrE}>N^nQ+Rol3F7l&1j?PmL2ZEb}Id&fo5y%)fj#e`0(6 zc0DsoiuBxc*5zU~Ph`QQ(qH-O4(9!0Hk;g*BpTl17D)7|dhHU4qWlj1{vL6LT%MEGnL*tN`Hl(|1qiPrsOB z`mtThyqCupjmU`$Sl^ffU4w;kK&)WSXCJ<{I`BVu!iXKx~Qb)o;Ghq z!3_`|23Y$k`#}Fb$A8WR^7VYp{XObq&Zh!JNsVP$m2{Igfv%e1 zoWrhlP~Uq&8CI0i>Itl$Q|l^0;gQI~YihE)0}T-q=W*r#lown==`8ZJF*)DDim%<_ z11edwjZ03FITIIGm*0JH-T1jB=r<&EtVckHoWtD~H;;HXx!Fy=cX3tD_uby42 zre@Xj^z+PVb&Q1TO%0HoDe-n#a+3L?FOoZiC|mAli=f>~tsXEuRECcUI0CkDdaNEW zLh~#O#=~a$c{=7OCkoew%e1-%p6?bN`H4juE~-5!U8hjgjRQktUifBB)KXVlPxdrb*Dy3J!k2HO{;!E?OopGk%>#6v&axG;Y^20()RNK3 zwWmjm<0EepL}sO#mp?i|O?FUk#%H-q`x5duXb#Q&z&JYHi&W}aJ8dd3%8wO_%W{X)D5uvYc>ZQ42GHKC)yj;s9~ z`Ve`Mo*lx3qNdwx^ku!M_Ff2XLsS%0rocNqoh#5v5`m=#IwsV>wI8}UfILlt` z^73i@?(q>U;XV3RjCl0!!T9#|gdTpl28{EfRO0=8%sIZ!wgqKPuZ& zJAUR9p}26smYZl0K&4DC%%(L!!*>h<4V;GLk~9nKIC!06_K@Bq>Ay}XmUl{vIdvg$ z7Bx)f7T+#b{#cNBxuHYRpume65d;PSeo_x*;Wt7ECK}guGWwcLH9{vJ+{yG8ATK&*1+qj(Lb=mKvIl`4gTi9^# zZXf1-xA){MKUCcxES0F9xa)N5$IC-Hh0uyT=uv|I?U*HLySfuTw@pjKn46^)2JM@$ zX>S4rv-|yl9&-1QEt$TuSxHOEArQHe$VbL(pg`p1Y%zc&V9O!b-hA|#D79GYj+jwz z4qk~7Mk&b7HnU0ZF&ZGXDtw*5QB?Rn+}GdQmGyC=gUj)oK~!n;l0 z_pXA$73Cj!WDUtl336O8kJ~6oy~G%+g;J?C;Kv47;Jcy#3r)C~XmS_*y_v^iZH+e= z%ni!QDC>vgD}ZjLq?5=?#-zMzp}6k4aC;_OBAg+Kn03^QJ4Ud}YDn7>G9Hkn$B}6i zOoLtNeUE_gy;|yo>~s!|fii>C>YM<^A8XK+|B)C5zVDqmjmUi$zwOk58^B2HApYHp zrLxkrjv$O^2WPkIof9FKKqEgWNcmDDJy(5{_>Km3$&=Z%gW#)rwXmK}MXQ(+vrU=T z^);%wQ&r?g5&D61;%cDVjB=R8B=2GV)B99R2(lv8E zzjQQ&QN_3rPrqTA!(hUl(fW%I4R%$@T*w7(OejRS7t0@-Ee~dOtu}q&*B45Or*)?k zCp+#Bf=j_~4YgTzcEB^_^xNmx_zlGkmEdvji=py+}Q*m!$62dpkFB zNvc-1^a3k+PpW+7jklV|wh%ltIj*$1F5sJ;%$suBO)ATC-$@A7p*iO^*!O(vo}3m8 zvk%{WI-GA|&AY|M$Z`wLy`C^xpj^R(NL#Rl%{~aU*f^1nU=Y@oL52Okiv@&O^+@V* z<3sVW7^C81mH7gty||e7t(bwDqIvi{3VKn5G4fWcqK;n8I-r+v|4$9eKRT&n@B=sn zR9TBq=t*sQDCiyMC=9}k*?>Y~a68sC0)z_{Daoci=+znn@XXD4r;klQ@0{(0-KxP@ zH;bs&sR?J0B-^rVR2%SYV)p0oz6!M8%(C9>D~b`mp=O#118TMW2=ndy%vx)1P~Swoh#4SGmb|AXQuFC@Kj zKC;g@9TPSmt;=GmZ^r_P^p!RT2P&mAk1lR;{=xLD`!vJm4K~+68`r|U->l#u|1UvH zas@|LqB%Myb5!!d6L;+*-%}HkO;D0g_#or?%&aho&2hC+3mzVX)bpNpV}~Kmx3RI7 za{S>cf8N9;PJRm$&6Mligvy;PR~9S!d`EN zEt4r`r|T>ntk zT7-Yh7m49eF`rOKY&r-lFCRyOZay4#K^ySI!_r^Kz9=t4wcWmie{<=P%p-}yroN3!QYdkB~VukRW&YQT@-`~f=~%1^&j%Uwg^6A&I@IK zZd9Z=14E;QT8KbTEvtggd~19)k(IPez2ddK$XUnUF=nf7Okn$n!te{@OF}~$odQo{H)W9-Gt6!gUt&1LRUHVrx@yXnXo84_ z=3X54T0>NUs2%54S~B(wggCQBTw&}28J33I&tm?6v&vGe(Vsw- z<5H32EJ8HxAYvPei&;$kZox>?0o2!$W{8c8V`$I5(0XS@{cQBLjSER+`F-+~NPc_1 z?}!a%wI*~MlXjTs34Zh2V_Oy)CeI9n!3Kb&)7Z~eUcxgktQfYuZZOl2VkfZJl+*pc zD+LD8meIgr5(c-dnxzV3mvbIr(_2!uj^5&X9QFbzX1KSzpAs*h11RrsjV$1D z-P(Wd=hQojM@7IaYHhE@FOAxa@?UWvY@&`|7lY$a*jZfvCm=j3n<;E01JulCCsD{o zYxfi|%-*nuTwA=iVPqC5qRnb7sbOq~6=qstUV!>HLVfduM^c@-2JA0|-B|5VWFYnO zWqF~vCMem_0ihy8)KQZjNtJJOSNXFgXOO@t%0f@B6v{h4`O_3QXN1|cPx^`T^Gm{MQ_ZI;-khDEihKlZgN;EvqlLYQ^i;_%J$t^M)KtkR zfSEm?)HDI-|5Y5NWXQumzvxeuXiNesiQh4cm(!IK8xF)fSdOU{UaD(`u!{*;8_$3v z*c$jk zQd_8_5;D{bEy(3saf^ubL4C+yC-_VI0~}I^2-Nc>)QDb|jbmE|LNGb^psqjDP$M7h zjo&X2%EaWMPrOM3N7F;V$|%@1{!MlYc)~CJmeBRoxz*GQ1!^#7It(nRp`i&}Mun-? zs#wKdsZ>AB=+xnh>SrvwC zmR-ECm0G-*FjJ{7$FTx6z63!y_j9Q=z8rbpv^KjW_vQ8>YZ{ZtGIgRvlldd~t9S0; zc5ussTQDq3@%t_1sEdROMb0+^%> z#D(jvGz+lHQ^pG?IpIZ+%3=Pg(r8^u@|~Es080p9(#vCww-s<7+>y+Oo2mjR*0|R- zT-rHb1-^Xrt!p0p-?@l%pjG*$3^Eh(*&@anvr$=}nOsEE=kghx(K{>un2Kc@1R-&l zXOk|(Ol2B$^}Khj!Bo5xU}W|Q=S58s_{m)ZXujn8X7NZ+Je4pw%-H<#MU7#FZ+lOy z2y9#+eTN1zU4|0i1+K!`;fLbk1ujEb;fLbjsRp!bk8V?W{ci|HK6>lDhJ>TqOK$jE z;8B@zDWD?tif{LNfaUk{8QpP7%8%(8-FRG(+TsTph1aF82XGfN)eWSgIu|9g8H2$1g02_04REnwAlQd&JDU6eUuWizyZW5v@DWR$%8N4CFMeaY$SPblB2!!PH zTm0Kk#7S$R(PtameM1k~4Gu#LDS1ppcwhaPkY!AlH})++ADhmr>YoKLIw_S5|MVP)mw!~* z$b(rNyHbwr+&@G8^h=Sa4kFSrp{n$Ky|z9@@LQ8uHLY^o>4ZBGjP z$_o}rpPBPBnLZ8ALKzWHSCOEh`nKGTUg5SAi;BphI7!E&Crk)iif zg9C=2fFXIid($OMR)x{D02=|JFb*Yf+C5~O{N;$qP{44dfcZMV`?D@Ah zU7f-NTaS>rJS~1aj#$ZT&dc%nK}_*1zulBy2hFL|T)E1)*SlahA!W%Tn)YKkgh8|` zLHOP|2Cwg^8XZowCf1|pT;UE5w0hZ(GV5fzlb7fkJp!$wCAL}0W0LWS2TBX=pmVRh zUAH?M4;(jwIgtchGL@ViXDJ2KJrH|(C1^(_#xe<|Vxv}n7;v7s@f%?JSfKa(Xm|FV zDfB8htOiYV>CY+cGLaXX(pP6$d{`Hg&Q4%cHsZJiPNc+Wm!+ZBOkK7$OirSdJ$zD# z+r6k}i3U^Yl|p`e5EJ}smnzB4}{4LQB8g;i{&-2@Nua^DHj4}yr+U9## zXZFCGKe-UyX7*3?+rB8;pj~r{6|efsObkBHMfy3K+88IT-*b_81aHeS50e$6CM=sk zZ$*K;+&`Kb78$I(5TbHxETZyDoS(YthBsj`DJ~=>y+(L{!_vLb?)QnbedeHtEzZw95a!nYK(Q>?fCfJ~K+tf|JUvCMB!(kfl9Su%eshK_jl8h0yJ z@#|11BGZzZP*%qvaXTZYR*d-kvV)60*Ho7%WKcYch1(wyL5OULdf7>*8v48GW&mYh4 zZscxT)ma+@KhgP^48yFmiH9Tz62Zu9YSPv{KfGJ=8PO2rhl68mQ*L{z&CTOOX)5*t z0z@lob9&%;GQ7M$?@w1(UmqW*w==%)kHeSo9>~UfqycrHW=gCAk~oWzm-J;=UoVlR zSbSJ$z&Ie65?nV?`L3AqiaG9RDdXsHSCytL&?eQw`#&*JpI|Rje$|^W*S2_NtSDmJX5ogFF75L%dNGeL5K{);a-x( z3yz+s>gnmKDI2b;4Lnodi2lDut6LcAPa}1LhDNJ`yt$3kI!~COX$L)AHuj=0grqRx zB=XXa8z7h3=p`e<6smR{_P&XPew_l}4lf`3ySwnyWZ#eT*-QkU>7c4Ha*-UC#a_E=>|G5aZG|C>4zENkQ2Q23;>Hmw*S}~ zm=PUoLvx|xLYNWBOM$PL3D#GbwssBZ2H72K-n!aQs4j2>QF?tFkXnFc=Z5U~And5- zt5zOrR2V=NfDHkcQz-ruEJ);U%qa|*11b6ZAfVAEQzrz+KiL}j7$Vxbg}~hH)Uw_{ z-6Ej>STIxEFwR` z$b3kMwgE(0^LZ0@&}99=`%pdmsl;|VF>2?1dpmxT*OTGkIvx|0fp4}DtpCjNF?)jq|#I($wq}r zLLCB1sS6zmV+Cgoa^A9M?DyFHy{6U=($(b&10fDv+-xyZu)4xkHJ1azG>H0vGD~V1 zA#s@)seFI!)VM*@5Hg56jLr&31UcoRXth46nyyqG6gwPKf3Mxfr0L>C5lEVFIu#Mg zl2QYUBtE$5(f=V|S`=J+Rv4m^8G#FHmL*={;VXzd2-l~7Qbf}W4y!^mFhA%T49)8g zltZBFUqK-98iLN!kFD2Ti9lk+Bh+(aZQaJ4Sve4a;B;;l-fl2VNG@B50FQN_j5}$Y z2Ks$y!Rs3TGUK^TXj{|`wIVp~{n(6+q4hG3_H+i!-f*C5YYK9LOn!r^yJnd!!uE0nt&=RimO6o zSwJK&iFq1=Lxf}QVSMdKOx(YyARB@5JbYcNS0pF7xiW`xYznF89-L9YE?&b7?i9MA z_f2olKnvp+D3p-<^C^)+X;t;*koCUs;jGzHQdTb0Uir0St3**vAlwl#3vU%T^|B(I zkyR14oOEh#quhb3*iB{on~IiGt1)u&yQqyL$9jheRY+ofZIvkuf=8X5lfx-6T>bIR zFP)UwtcyQIc>#6izNK0tf zD$x#Q!P;lDzaN9!<_!frgCSqNID`y%fqCFzuK}Y|-Yl#C#Mq{Fut#9`C0WG&XXa1rgt}+?sS3ckY;4c70G!4G*@*1OsgoTEul%W# zmW+Z=QzCBKx!OO0qSMC5%buy61~Q2~2KPs6OxZ2!&^olL%hv*@oZHn<{!iq?Cu;|S z9s3cKv{ADW#ZRVUwJFd%WShak;1?SiMWsO_n~}0&+?+W*NOt`iKG@@tO2ncbvmgq* zj&e${Cpkxqii|cL%*UXiA(vl@kW`J6jfMUp(3ot*^M@$8{`#dvI~Kfb7RcW8V|F6Q zW0LB(o)a-&)#HP^>q88`aMW)gFI%~q11=d(L5Y!le}SwB7o_D`=Jh~-^4p9wV9BqI zB~V5EX8Mbr!>MCJK?$aAX#yiZCb4_qTxJs&!=fe_E)u5qiLX1~5IOpn%nHN1^Q$s7 z$Ava%%aabp?HTRKVK<@X_tU!UnH%jYwxoURzVWX}^E+)s*aO?~Mexu%0Vjv9hx9cU0n`i2EHTAsM zkmA!KlSHJ>46obcQwMy6h>b>JFr~)~`0wLaQN^IwXkICAj<-1_BOb1gqy6@1UC)kJ z*DpI~p>D3XJ3N}lRCnhhhRh%_ZDS<{XlwF(1@qg1Z|?GZ?YB72+KsMy2*W6AEUMN% z(ioCubPN!6=c<3dRshR!BkVFNS>RIB%+he}AoP6tg(S(A2GdDt84{tkzebp+$2K1A zU-X_N7#F)2dC$5(KmEn&Z=M8@9XUVIT558T^Wx&u__CGg4o6u$nj3kzZGbfuUov?W zgzuKhx5k6o zofYsP8ZGsdH_~?ZyDU;tiYXB5GFTK~-P=kNfyHhLGq_fPt8)d4Q=>4|=I2_keFHd7 z(+*)B*7WbpWE6f@8ufEct55wxC<%d)UA%rw)dr$-bRJ_>t20Dj9b#3eRgRQ0(5 zQe<>B5~>iccDvP}OUvh|IbsDt;-oBDW>4x7st3`B+@WLmw`_<@GCw2iVxb1RMczDk zi$Fj!vbbWL-Yx(1$!8iA|1Mo;wIaz=A7vQYOroN?em#61#pG z(h-!R0jPZmK{;*^7D0}tI=B1t8gl^svadvPlW}30V_261jVUF8X5RGgZqClP*Sm+! zS9vm9Pu>wv-_#?r#uH`fYars2vPL)~|NXLFX|k097yzo!oz-RA!2+dE;a37`VO;^H zZ5?s}sprJZk3*e%h3cpJX+3bi0%L>Gy#B;#>~m(6HsmUms}L#M-K8sQtT?Qqn@=jC zVxzDu&!f*%^>3?Ry^jeK!c67kC4?ZRC#mqpVN;LI;%7KoWO6O_5fz~`?YmKz7|?YC zgp& zZ3i&1?r{lerOFGCOAFRQW@PXuA9qJ*+4axp$*lYqGteA-BnBn?tfWo~p>)hAU~9tv zb#)vSL&+^>xYvJEBFe&X@Lwy0>M!KDGjdVVhhRDie0<77_bthz>{TxQS$7*0H(aY0 zD0F+kWp$1|*5{l_*=N`E4hBaWfZ!0P$!lh5u7`h6i}hX03`9pd<^h~o-FU)T-)FbQ zRRg=TiMkH183Fe}L0zx{OfpgxCiJULtjYxrO2zu)S3&HS3!n1J=q1)qvpmLKs!F)K5 z3?-$2v^}0G$dEDMI$UkQri0nGTq)vyE}aURgk;cS0%eA7LfzH^hL716(<$-MW#P#w zWAO75WKk_>&h=i;bNR)n9Mog0v`~YzGYj7HJiDu>H+%eov?;5Q^R7xU{YhNNw;%5N zn--Gjt3e#*C+i1tO>;L>ih|D zeqK^d&b}O3Cd^K3rU`$EOzi@ttRa-W_eUhXK_=I@gEhKH{Mti@pm1>6LeX%k3()@25=J@?GKRf%gUEA(> zKQnRuLgs#Z0j0XvyswT9=8d_}CiKO|@myVQ=)i>=6H86ArKi{YhUxprt7>1(3EoJi z8pg8NQ?sil<9j5i7?D1i-xVTlvem_Gw@Of;bzR%9v&dfIvolzn3^ziiCm*dTZzJ=xcp&JJ&2R9e2?{{2MJAx^y z@MLma^w@55?L4mCx#Z4M3nl>kpncBcteW(gSa|xJvKbM^NSHo_ln6^}KjNSaH_Fj$h}$WYpSe!WNr721ynuTpue!!;lf6Hw?wr{-NVO}PD$ zzHM}9VyU5*klgPg?Q|)6GXZ^4Y_(y=)F=)-9t6KqrzqB7J85r<#6J;zv!;?2k}Fx= z{b;0V8ohl!AOa}XaSjz{#A)vfg~Yw`J!gh}9vEZ6@S`=5EkRv^jk+llP#DcqlcRht z_B*B1(kVdLd`yx13x^b!ci)^ZOGAn0 zHgIE2M$ih9c7T!dVrZWa1R(p&MJBR3;h0idN*p4o&xFxFvYQx5Gf*+CNQHOERw-?) zUNp=NN{l*lvakT1Bqz)i3|*tvC|YM^8cyz$el75fX^HAA z`v7ToSL?7XMts)dRh+F6Lr3u?A?*C{K@wF)qDQBAQ{I*#AU;Q5 z9dkG#%7PoD0*7NBKOTLa9hnwB)2k9IR2#fVEpH&PG|Dt@-WMo_HvCc2Q`+#^gNo2m z5(OvR$bQM>N~?Np_eYqsLp92)Tr*~@e8ExRGJaT5surbXjoCAzT4}5FR#k+9Ok;pJ z;-o{I#USN6eutb8KDP1zJa%m;J_I-|thbhal8I0bcu}hIHfC5ZAz?9Wvnl3M|4Sxc z_=1~RoGtLHLYSab(5IrOp^vhV9IBC0&QXIE`p8cJYxNE211(L>#HZV5>l5o1VWHY9 zL&=q4RyT>vyW|_J`(%H-HV?j)OgsdY6?Muz&ef9-A916BUJ3iEpNCTta*m#Z?4QXN z=!735^Y;X~f4P|=Q(nd2rgCFYW>^r-P`E;+m>&+=(6ipivKMtmGH%h{R)JthGDuNQ zh%9ns9tLAnAEs8vdJ9nE$zNvPB60o?81jqTycM`b%B(;gVQ&>u6Syj@ zaDQ`j7mtYy2=O2OdcdX|onZt z-Nd-Iei{XkSCnq5f^u$ndpa<_hn3LOy#72aL;GEqeYE2|57M>N!vkT2)0fPHRB2q z!n(%lhjySqot_<&sZ9mhr%V#6RN9XSZ$~enQ=RYIyf0kqRjz-dJl3`t9nn=QAQHRf zVERFfnWKa}bi&;a_kh`?MINZ<$se>Q2v^-QiG&J zKhcA(it!m}YUemL{%0m6mr-1$1i64nXBpB@$CQzk*J6a{{b0igB^;sG3Yzp{?0{(MB;zr2t1x(x`W;U^Ml4qHf zAt9%kCb>tEp-~OauSrSJIlwD+d5;w!So_Zd?AP5fulLt+X=;&w0HBSnlk4;Kt({RU zDvW(mbXbMo)d9uHOoMlqxtq8?k+b3s3k$UPFw>+cQRJL^-+@{_70B;G@+tv9;Jfy^ zo?f=S;+$Gko7-mwM2*?P-i&Cb)D zrbq1+6%Es$i&PDHWLCE;M5Kvr4>j7_+38(?EfHmzs}-l@Y$0*C=;X-C1pg$y#lu@M zm>=>P$};u7vL4>ky3$euxsxQYEDrsct+lhC5WE~!#ZiV4Q-WRShXV{BA%C((HFX8y zX{^ddiqiQvVjEg_Z4M3 z3SIMdDw95R9CRyQVQAE?<;`lzFbsbc>b3EZ<`v|23&LbhjG4oyTC(oF@%zCu;TUTx7%tD#i~N&g5u56Q6OvE(Ne zb1jbMTZmW7_b`ml7~Nhrk;%vOC#qA@EUNfW*aOWEj+pT5fFmX%^@=0tP*)(;W=Ky$ zGx_T4A23!1rtnSDVu6BC(~1LKZPT$CybpX1@dR?LzDK>jkh0T9sC>@y(D%>`g_8tp zk`V`MBNedY$53Mg_3vSTmdrEbrrE34NSGrUqXQ0%tbDlDR21)rqZ;prdj!){DL>lc zK|YeFtdjn%xS z7L*2lpP|V_Pp>%i5&D%ofYMfHykH7^;SfzLEBx_ep{r;qBKONN!IiOFGCiIQob_83 z>y`Vws}{r&F$lvCo#Fg?x&Tkoi%9xaR(}Z3tS>s@jd1e0%bX|ZI!NkS)8w_L0KvNH zS?Z3zgr4-hh@Gb+U*F-giGoK&>Wp!5IDMC>G7z*yCj&qfMy-kdIw^tBx~G1PW4Tf7 zk}%sb!Q3X{nRwE6*E}u}1ZJ@C2ihM;5j2;=_bxKUgt##^lBy#}#>_Yl0pznW;|l=t z_S%r9ue|;#ye*0UX;NNDajvfRGmYFoHVpFl8LniA?H4MgwgeYA0#mhPzX0MlNe${I7S1(w3YY11maVH^s+6H04gnm z*0Y{J&L?Kb*lHMYv+9z!6FGFLPymT?xjm1xt5;s^#K$(*{(U^Qq;NnICiP?*F(!=h zxOo^ti!VUiyUHT~gELj#^glo&N4R@cAI62&*-Mw};- zm*I)_=UB|&Ge+KCs^fe1`F$`%i@s~v%!|F}m~CSl>KG@91oy}?A^|(AZ}`Rbdyj5= z57g4(YP9$Tl`HqKUti7-BqnXu-FP$!!U-v+*7cIzSLtC z=3(?lsji$8;cZ5TzfxoaKT`@vihdB%UVkQZpY-yu8>Q*X>?eY#Iap0U!}_}-xcyNF zpo)zPse)E-@q-V&Yh00cQ$BFQSi3XucM7*WNNfd6;P(WsEEalsm>J6uG7*tNP-mO- zBz*mZa)eWjvGToH;+CakAh_nd3Cg(gM9AY<+cg|1z*w>K(DN>DFCS@u<%N-o5dIo5 zh_@|h=25tf{5=@7dO-#r`9+JwG0kEYSYP&+OQe~=bIZdktSXe@NV30=aVn`s&x%Ln z>Q|gui?|3KRrn1TbNAYfLd!oHDp2#yKS?ITvDVsdh&3!2Vv{=C?QfZ%y98TWTBFLR zi#vb7bmte;bT<(NsQPtsJb8&B@AG2$+lq?v zb9nTFg@Pl9e&)V2X^uiNg5Fo!cNyC>;|p)r7!gh z{PI(#fYLA!hLEH&xqhGZ87m{R8znwP1u_uaCltm>L)IzlJQXVfb-}^U-F@DN{L>5~ zrK=WG+4t(*k%w|%;n_4lh4!H{V!oJ&L^@KT0N~k0KyW4x!)%|J}{p}S; zN~bIbXqoObv%7d&%&6`X;~o%DXqxcIxM->vVDT$0X3kj^ls?T|GpL4p=2*`SkKvp! z$U9t4oYHsiMRh9ZrFV<^3c8xz<_mq5>|Q{^!$c^*PA!@OeL8mc9Dme2F@QirwE!~^ zxz&Mw!D3_q+KlR#GYJxer&dv76(DV za*_S@OJOnOkNg-xU++L#rCASZyY~UiJ2w`YOYO}hMJaB=`z8nubNb=sP#(_znz3D`n!$65M^6;p?DmA>6naaaf<>b$k3XG z9VJP0FD*B+g)i4fdRos2Nxc2J;g*O0U-MHj+!AwtuHBSvs4&{@l+*A(tM?&{WS*hI zb^8B=o6`|?r=EP?^}A?>T5nev|9Q#$(LG0OXSp&C6*foy|9t-6NSaxvyB2<8Z#E1+ zdma|~SO(Q5lH`W!Jdmx*hXB2UpV+>Ab%*&|Z)+8QN~TnH0jF(RZC!18?_mvoFDUO+ zw?^O|>X;&6`4iSU6O=YA0*PICxoV__-}M!SMEGBXeVjurG}P=Y8irigTWzuNt?Cgs zt1(SGns5_6SEr0qh!(F?NeYO~ragzk;*Gery_50apf0(szkAP|s#m@BWo6vC{$OV8;>oeF-ZkimA5`8hBFsr#|H@TgQOo>kX%fh1-`%+jxs3 zL@$mp{ErUsbL>Cqw@UMNCl-K3;z|yi{6?)4iY-4FKjlflE#|xwk>CPx^L$ZXQ@B~Y zRK_`d-z8fKlFLXW8y5?wMUo&>O`*^;_oLeHu##3pTqA9dz)X!44kqX-HH5MiZceAcJsQ08y)g zC1cfnRq_{ub+xQSSzjVI3q$l*dS#2JcwwbcY5qrYp8gU84rL8j1;WtLu@C8UO^$Hx z&G>_$RPzh1XBryiB#PE?isgb%0w@;dCw`Tv|-wtTjv7j0OUZ6Xe1U`oGRcH9gfMMn(yksix*J*@qW zGq%GR=2<$gxcdN1G4KsI!|Fj`>Pdj7M0C#nwrUKw=?1oqH(N3*pgoP8;^CV&77m}{ z0=v{c&GRK~^%+5td)0U{){UMRK6~sc3h|)LDZe zuf?f97uLG%;=S`}zS%96zPhpQ zkK~Ggn(EXX%d}V$_GIWOKEDtb)J$hSA4Np2__R{<+)`aRmn*dzDjIs@B1iFEAj$pv z#XC*kKpx=Z?#sYm3pP{#OcXhQH!nM^%Kq&0GTJ}af5i#?9H;m99xYWgZXQ)QQ@B`5 zjAWMx+XncKEdO@;-s=|l69v>^{@I&UQjRBTq7E-{C584L=1)SHlXk<&dYs&)r#J+In0(~-?2HXSze$A%QGOPHqpA8OMw7Wi|7{3*JdT;FWsy#9N|8ahz68JGD80*v25PRFQJCNO_8^jP1U3QNOo{L=bQ#37 z&VOt`jg-3Gojfr#|255}Yo8=MdaQDbk3wHdeN^5Xu8avf3*Rsh<|0$F*lfncd@mP! zw0O??WWtlV2j9GIF2TF zb{S$q_adv&xy*u{{Rz$%eug+MQM?B0brYq@Bk$)iyK1@FAiVnfGVO%nAI$pSQMpzK zX5c)F0xF+LZ5J&%H(&aB(#iu`RxO$n~jCPVR~G&@=97Jr&?RV@cJ1j{`3ALzcaKDj|e{W_GV91pDc%)MsxYIlAf z(O|bIQ6J5jXl*;g?A4`5wBH^zW%ycX1Azow!y$IW<3@+9F)l&;=_;r&O>1o~u`cE4 z2ThHpJ)1Dv@76P%E|$Ctcbeh@_y(_+S)ql0=FG8e6!82j*yY&<%f5>+Ks1@(@N75u zl^gsgo^V|LOwgrX;uiula|xMvdn!5l_3nLklRp5T4!Z<|LILs@%$4Z$1c3h|LPVk)*-;l zpk$4kCWQTpMjA}V3^l8Fml!Lb>HBxiCKe9FDSo4iZkvHmko!7Rr8j*Vyw}bBHGD}C zw#6)OC4nR!US7G2wVH#SV|H0YzA2%r%@6d;b=zXQsG6;!f}{Ro>;af%=h*J~99f$~ zaE9gsLHZfpG$a+R@TvEAa_fMpz#l>R3~lt!!_irpf157L#gohY>$U%A!M>tw(-bKL z_0ionVqe4VH62&hF--|i6|&TfvQJP0eiVXjoA~ zOO&r_LgO=NLtUd+uO~$<#|-V+ZwY)zI$0rkzg_DFaI)o0pSN+mP_51pUQX z8a&Y!YuKx*7niwyqz>lWgyA&Kkp8%={PtiHMz4^7J0+_hS>f^Iqdk!@mpFNz_WkTl zg^{+Ojc}EW$67`O$D_LH(hHhXtA~sCU z44#CD)OH@C{^#WX9{L$Q z$<%E>O<_%apSDOT1n#SvQD?S8i`var2wCXmOr}Oa8{BF=SZIM>iRhHa9m?z)$WqZh zHz@-j3C5D{|6%Q}f*aYo1woscxysDU%*>XVnVA{NWyUfymljpdq?FcP-&(F8AAk_*jg0PcquKUuU*;&LlE;cQ_T}19yNGY@ z)rnE#9|?%&qzRbjQCtVVXka)q(fd#s!`(`a$f$M9A&iO~>rN>!w?zFrRjI?>di|6aH9$}-biRg zX6Cc%&1!b83@Vpl>D+=#Uy_~0&Q~4xj%i-#4k(}(gse~Q_qhpO#X2E)xE(E&zpl1p zC59$H;}T%fy47gcdA!3z^>We3Wov*%Qc?y5@D0I2G_;0(*Kge)frq#gdc4Hec}h+p z_15UDeJL+d{_{fb#}X%&te}?fGu@z!`kRxP`Tf!|idck=jXh$38$KBg6S`lL}Dc?Yv`U}Bs#aCq@;~qSL8!R zo1A2BFTf&O)3fd#qJ`OLX)uk{Z4_lQRimG3rP!4yey*+6?g3;M z>)ge{6-eli$B5phj)p9FOKfs;MfuNBJ0DQN%${>$_EN>{UZu*Jzc`(ej-rp#UCb1v zMYSS}a}tOJfiTiV6bD0KuntNn6RCJm}eg{BCA?f3v9|&X)!s&0D)l~N^W;7ezt5`Swh4zEK^)T!W2@#%c z0|7}YWQyT#j;|_LsL=ow@$d|)*(R4Q84^)Ux7oAeVVdC98u;Zvbg)uEL1WtjcD``T4EO zbMb4}9gVB$j$%9B4>NF0s~Ec_-N50Su9{K6{6?p8@LR&1xMhjVGDz15PlglB<~)tRv=EuhWwLvt|gx)f&#@m z6~6WRp}zl0zPd*eZ|s$&tkV6x$Thn8P0f?=7Xp(ozU290?i93TzD{C}xIWEs5vIS~ zWnx8=YbWvx3aO|X-n|A!o~`_z&1%HLMf4r=zCDkr?AqOBS#$ayWYc7)U0%t`I@=o` zUg8W(3XK69$!-6r_xG|1-=j4z>&~MEUuEj#7p$D?4klsocESzy`E(6YZwk*U_QzS% z*!C8(lkFy3VL}C_kOzgith%`saYLG^$sMbcKD~;<3&2UtpS*2E(UN-jdiWR{ru;yJ|Z&o6p?rsuL{9*wBzIyzfRB0v^&XB6e{mrrFRqtmJ7&xp<>ivPa{` zf^;Zm{OP^{IV$d03S~f1a2?`q5s+odtPj8&$7|6w z!hB&h1+%Z5f+7y;S<>B0oS2e{XR~*3b85Yk)uo)pb)S+^!YE>+>-Ub3y!%YPir*#q>;}X_)(+-qE6J8k`s-Lrqqe6Zt-4psz8PSJ z0S`(UGeSpjZA|?cp1@&h?pap*|95cqgJ0T$4|z)n{u=l zINtyI<@m8pOnyG~-<1?F+HKwTI|y{WK7CD;#U2q_NZySXFn&5Uto`i>%V;%u4UOAv z+!hyBZdX9&{m|nQKx&}dFce3J?&zS0CuJwPkB0+eFGnQqsbm|=AF4c*P0GvK&lQ`G z4|NMJI20x%O1pHoI3!#ZlT>RxFCQ zOOm&X5RUhv0j2v`>}N@;P26`q_-QBJ`?1{9^_P{iyZNPuKv?DUkx)p{+7zrKUfaA; zw;Y{Z+b{!v3Yh)_CkoC^cS;h8@WBwHwE0+A^7BSI_5#LgUFr}-w4hmS>lfBa?i^(QoJ7u4;~v@i^?O~g89R+t$d!kbQD zBC)3YO$3WgbQj4&I!jBvHFSQ;+DgNEllL)lbV$lYDsu4y{O6j{;`)o_6lE9FZ_S_! zZ0UqvWV_5U?5{4HTRLKw!fO66PD+H&@h7;e6uk2m2#r2Fag84*RVoK{Oa@FI+XACw zNrhi&cvFaV2f=Q}f&)G$S9qEa;4@T{D_(}9qYc+tvKp4G>dQpPLU!0gXunJMy6u;d0>*`^waA#8P{}}K0a@1`AfVe&F5PeMx8W3$E zGCn@fy=7(EOXN+9$~G_r5DSaDM}Z zUx1g~o<5IW3{;zS8gBoUcg&Z!=Sz3aL^wPYHpuKLx4VXhETLusEeB#%Jnm z;u8$A3AFKoJ*+cw#9`w4t?#!UXpeTLvEBG2W&fm zj@$rG4OQ#<?r@wX`a-kInf^Tu}Q46MB#BbiJy0az{aGm z(@nJe7-DV-lu_CEyTX}Op)l>5V1Mk2gauM%Fd9oISk2TmWxLM80P-nzSuGW3591(N z3J=vviWI3V)wu*pJuN75Pd$^d;7TpEI%{CO|6%pnp$^(0E)%uEP5`R_~zHv zz>*z{7a*T}g`%m5O^c3J6qKeu6`1S`M-IKB6Oh z3=|Pq4z(&KcVX8c)z)0;0?#kXr)b5IVNp`SmOmE72Ogbe=>0q6G!UAM8qv><|OBvQbyW;+wF)iU%x~cc*xsK@!h@JF|{40UQCa$4sDeMhSw#kV z#cu1*8b z3-40{aGTLnnhut2T`b2B$n)4!u2HfJGh~ub9vXex;M)6Kx+rR@*cMv;_RLp=YhEb^ zvtKAxzCm1D1LYHfbqR&Qmp3S3RWEt3Ve{=U>j|jb>=vTtIL#CGjGbjU=wG*9F7!YS zUA{V%xvOywz-w`KHRcJwB!dGAG%mcxwmc9^G41758F3bv;(WO!OGn6PM_w-{;H#1?%$MS6mvd!#mGQ?YFd2(0_ zp2B5TKK2YRS*D)|BMET9sIUj=$iUoW7oLCBN zr*eTOg&HUNS4>ItA-*0?7_wi?zk}3PuvZz4Su2H&{9*fph!%_`V9WApR?eYs1wvsJ zNH+kx=in7sZ|C7Jjz1CR)vC)LxWF4}WG)tfVPsR~89E$Xm;(QljgSW^-ROmkI#K{w zgT83Gc*hM?_?WINKX$tPxPR3+W&_v_FDNI7r>kXwVc+qJLdDL|o1HMzSiKy8my1HO zFykhar+8OpRd~uLf{HXy*kDp3h5j^P%Wp*LSU90Rtoy z?KyPLXi>M;DE?dT(D{Dkd81H;S>5o1quYz3zD!Pb89oZI9;P!@o%<6CNkK`Krk=ctj3*SY2hRpKTm1&B=p)~9{$lWal4f+ zskl(FF`9K<#zv@TE;Sq+lQLM*!S&?Sxq7>!isBU|YWp+Q-6s3^7fs$FB zJ{KBV4>k*J-YY718{jth5aoEbsoX7^M7nC~#}GOmu~`2gk^NU|cX`G93aVr?x%cSl zTu!aJK3iCFR?jkx*A~8T2+~%xF$fzD}p6EWkU4TbB!F>C1 z{qluUb|(g(7xW{Y#9jI9oKuKhd26yVfY^bSR{G|dG;F!}%s$rTYi67_q=DEzDy^$J z0inPfL{p-NMF{`P)ad0#!MJpYo=}J>tQ0hV9QvQ$QRE=bxhU2vvLK}<0u%Y30TQ;N z44b&UyrMW0mQ57w*YT6eTcHzb3~Gi}1iekWIvXjYW*`s!;a6&Qbck1u*N7{+@rD|$ z7ve$vN+ePCqeJsKd@dmUohzYk^IhZtBuSO5^U8HOxQ-JQq^>_a5Y7SbJ@#9|q|O*4 zE$Tu(OzfP)iv2EMaP`*J1%Z$BDBY`vTg$*ENL$TwGpAvhqcp+)7X!?E-5lsI#3e>{ z6Dh_U9W)>gcj@4hBBziG$OA%&C^S1U3%PeWN74zcMQ2p+Sr%2rJt9mQm;DF&51Fr@ z^7pdnbI<*~y<1(}pGRBBNIzdtg7)rTg$|+-+du~zR806nrsHtZC`@}sluD%gWQ(`e z))tNbI~M6BC=rGA1`rcCl6RN;z7AD%lC)a2JxY)YIdsk(=GNKE_UCqorpSGTYl$=~ z4DP5lK5(RxuJ)3;I;ncM|Lm4 zf}?s*#Cp(Lf?~>W^ps|}k}E!|_zNaDc0l?4n}CE3t$ySu;6Fyh)+JOkzBG~=R6M@2 zQDkPVvX%gRR7qw@^@fTq%2w2FsK$JL!Xuw@i>5&-)4{m?TY4P2&lO4Y?{)0;#dp?! zQHbS~(F%cn&fUK##3c*aFDpaY`~Mh)v}W>ea1hL#twt+K7`d=rjVJFRvVU3e74+N} z6!rPU%q5N>1Ap}Nkwjmsb0}@nN@C@Mi!lq^7z5=AkKJ4?WqM?Kh$%bDRQpQ|G^=%sQ=&)jd#ZK%PjY2CZ7-opNu5+6?_$lWGLLS`m{Wk zCU^Gix!IO#>8tFrt`+}x)k~e|&_$~qFQ)(>^|3N6q?amsgL(I9`}1GuATDO+K&qJvqe@3+i>Vx{pSOH?R9w`6smtZBl_+JD3@Hw#t|W!u zrVqpjPG|DKvt1~Xt}&x+9G@DL;S*;Ij_(UY4?E+~Gb}{de9BgW`h_BDcA{`NdnF8a zAr;DAFdnB>GJqtpl-KR*4mB@w9xV~4*vFEc}Qdxruqw%9{R!sJN)iXCX0rP26tu2l4E{wXA@_N zSU}{y6Det3Cf{o>I}}Jh{M{JEDoqE(Nr!ysVZPwPY&7ush*V^>J^lJ5}1_{YU0Lt`JK@&gkVX%QlmPz;*x3ZI6!A&=vdL&-x{TV299B&QN7`X zH~V#&!=>7iP0BGP9%F`&vP`|A4zUj&@<4H~EzJf;@ti%R$B{}SKq~_k)YrD+Wl<3U zFmAN2?^sMQbOoVZxG0LQ@k7@se6+?7=zvfY2?Go~t!VYuo(|E^|3`sj;J#sbqy3*K z5Qo%s*$&{KVh2TSYqukzNbv%(`o0j4#1YxlAD=41Zpa{(v2B`FjA*Lz%+zFEyKdp_ z?9hCEwS$cQ--$T|+q!xEL5V>H_}~gju!PU=TXq7{ji3pK;!OeUglQQ?m5nv-VZ~DU zruNXb|I>b$$Nklz@**Fbl#g!T`@6l@%S=MUE#i@dPqjQr-Nx*7voOrjAr>{DEp>_(eh={Wskz1~r-OD|4&a0#L$j{8p`3D+_*A)@$y(;p^NJd>OyPD6tm z@XBfBRy&u^$`{)}E#KrbhDOtUM!H$leay;|KXj$dEp{#l|6lw8T4h;Ue9dm2H3MNW z{kNHptKJB|M;DE56hfC6G{VlqZFc;|f7Lt-yt9f1I*on62qr*VUg_K9p z#xreko2W8nop+~s8Df({wKLwSZZXF1cGlljLSUGNNoUAt37$Ww`|Ww^_otV2@d=Sp zD1$tsf?wCiB|<`hvHbl^ts2TE-d9fUqJrlHBZ=7oQu*7^P>+>YNxD_hjQ7+No~$Z` zKhbLfubG6D#fdU!#{bkGNwNQ3e}tEnAEk3XP_CF z#@UC}0Ck8V9-t*yJ3oC0aipQH@^a=?T=1%#p=WNY&8$xk$W_?Ac?aM_;jfcgZhkDT zgmpD7vR6ik_!y^la1D=M;A2s1R@UUZWXvdzeOtJm zQvqC~41%U4_|;*YD?J%mH=PC3uz9kOQxcK>3e2Bl6_GRReJ3Ix(@Jc)!h{j4J0bo` zBsu>XCFKS7GqjHBg=e~6InJchIRU%MPE_oF2|vbM3y2=)@1e4;@3wO{2>>&xV3gCw zX-ycgc9RXJGzT6C6Qc<%CozNbqHIUdNN6bG(*FP5hiG%9WXu0`_+dx$|Gje(3d$Sf^{(9ww2xfW5w$b+_X53F^v95MYmw$PrB!5wgbVCZ+^M3 zg371A*tTAapK-cySvDNf5Bd{C8df%0N1^kE(#|svpZ>;#PTa2*KjAQ!O{o$(gvK^t zA#Hm*nMYC8+3M|c(hBcKT6B6;ICXC-N!+q59n<9TZhoncrHi=$oO{_m!{K=$OPSms zq{ZK)?acS!s`Jpb)5(H`{OC*kLj8XVRGMopxckj`DCgu{KaX!$g$|?_QSDkCYiQK7 z#)cXnN>Q2F-y7h|RD1nR0IG5Srr;Bt zkuv{q_v^E_=J9yCmg~{1H40>ABJ|5{LdJ_`+m2080-qUjDQgCcD;ousDI-cv%ooH z#M`EgxEU{ULn)W|S- z#W{au@#fWQhJ63%>wJUTkU6|!ldv=4wtUj4CSyO`X?8bj?)n$*(M8;XvB93-!nFYb zQ(C>d5yR*>j>UsOGKWb|2k#|*KEL7T?Vte}L%NyQUFc&2YLAU7UD2JAiqJ58WZmODVPD2k32C4PiaTifeS`#8t@iIDuSJ zK@+uvTFgqE_|aj7fzxtotwl{B&Q3#x>mU;^!G6sfGL{1gUShZC^>JCO$duFfuUBA`?sMqjiXg> zROXbz47<|M4=YX2p%pZ&sEOebUtu&v&QTb1Gt``tLk(PgIL;|6Gos4Z!P#@Jx=L9_ zutg~FvXwI*jqZ9uzJbhNw5>Fq-`_&EZ8E-}mGg>Nnk~a)SYK!~_&aeJnMMdV)QHV) z7e0qNh1FsJufp~X_1RdYmgZeC<|)tPj^msLh?B}pRqaG1Tw7@{WuEbQ>lO)VP$gXQ zYp^X%BcjBDcf7Dj)=_jE=e#v7g==Uag=H&749f=3eFeWOYTc7Dq?2`m3E5ie7M6;- z^vO1I;*Yv+SAmnUz5RUKK2Y~RfrqbCH3K8hu9inug-pyf?(=Vm~|J!Kz zGFUv7{RwIa4ZfWKb;^{+F1Xc`3?u<~+(kW{`FiO4i?fwlC@4L=s;Ry%3dd%jXAxbkTFEozp|ChRrQJqm2ssiboZ9t-P_kzEC34zv zYT@i-E$c{2VH0lIrK6J$iBMCpvaC<~Vr8#_R|%`X0LPc;_!1lcT8{s6*U#DsV4~~& zi0~f_V)3OrzBaf1bses}L;nbo&e!y>1@f^2$GUNp>=ypNXX#@7+Lv>^hVJmSZu*Z6 z`G1|>dz|&_H!+IzC%~MqV3dMJ`EaahKjA@5wWmEf2PNue*B zGvgmnEND~cb2be_w3B};eivveUi(DI34@4Vy+-a8Uzm@{o@+NCVQMxz)o1mO1w19B zgbE|z)a;G4{G}xhC(gnw*Jp;o`s>~AD39*3UZORW=%~_DOtjx+(^W(ncU07O8znab zDMms%3Ff5h0`{S1Q_*CxhRcYuLttm-SF=EIlg<>qorNlvhKB1)!$IQ0>UBAqAd)P)AX?2FOjS zZ<1kU8lJK5zN)(Hj%t?>;leM`f)A~>i`0r5Hf%0A;cU6j9$2@Mf<5im*O zB+v)PIRM4k1EyhE6mD2O{fYPSR1%w5z=(|O=PB@9{rki9X%FM))80&~B;khOhpVgW z7Y5S{~R{uGN9_~;pXTYNj)+d>QX?r`+CB%uE$u`#H4+HADZ!nX~+jQx#JZqu|pj{6vD8L54CZf4P&8ZjqM z_eT^Q#jrNyn-lpNI9QLv{K=+5tItqrWOsH~2(}fV-{P7BJS5F_ETU+kBOaO@h-ek` zNCWE07C8B3ver29`8G)cvO5)NB`y~hD~gZ%g1HGff5TkaHEK@9@9O%rzs|WJ(?yDA zRDk-g4278bqeE2I+>r>^0iwgQB}Q`RNu`zHfE*TC)XrNc0}jgXXNR(gZC4;9`f0ii z0)+eWls()5QCz-3LUD0o>7<}QINQ5%B<}4Z1?b| zllbuNMdZbQk{GD*46S~2Y5%0HnWo%+YwmOmc$wgyvX!CCZO={x+?%~wNL>D0G)SeujXAEylET{qHc4flm>1&B>nJ$(D*oaeYm|B zto}-udB5C6({mQrEuWBCm7;mI|1DQ2d4QAV%njL&IdEK`v^A=)5t>K#A|$0i9t44n zqfG&Y(UGm`@hNU7XXTv$r=JGfz70KvvC=-aq!SIUSA~)Fp8s51)Pt}q{95TgaogaF zQxdRu?2&@L;D_fc(4xuCgP&63n^8<=cdjsEcIRK1#tOaZB9y=KdiLv+bANn<kisu+#Z;wq zIUT4l=>ZAg(K2EJ*V@~1?uzL+>a+zx2b!A+ z^KX4Lp2@fA1)zDmOBI#q3|-m~R1 zANi2R+qW7C%k_cyfHmR!mc2ArGaM+H5gk{acG-6?LxomlQmvG|$^5NL1;2pNG|=l< z^vow$R1E+6axKlOS0Q2D*I^I5X?_W{-H70kN9ydy4Q3rRF;zZ0+)y8^_$c}0JF>>C zd?L2_Tn+I$7dKh#Wlll_tL!P!hDnG^sv)BSSErg`L_m^EeTTvW)7En?W;ACr)@zZP z13q_qQ1?pPv*&b)#KPDDd!S`uo6;alRSp}vsE^3zc%|z9u&Yz{_Lyhvkqc|8;Q+Zv zFF@whgTbP26aF}DhN;JGd z-ApoB&H+Vw)pZ^jZtU*^*P9*?T6FSkx)HsP>@UK;-sKTEVszabthIOIJkfiAQ1Q@VI$|&3NqwgvRYma2`ZuK!c|i{ zYDq2lfr;$#rDgTzVo-a!Y>o#lqFInHfq=Acu+vrnT4T4_($>OhY>}o4xWQCV_Y$Dn zCs9J*v=W6@rY>Z5UJ&`=J?25c%sk~;aM7&-3;HPng{gSSyrV4o>;6`IO2`RB>Lfv3 zOIN}Y;MGx7QgAFm$lR4Q6tndBcq|sCySV^*m<6-h#lMx2Nu%Xpv-UOuxx=BCHEf_W z%5a&Po^UKJ&{44T(Y~vPmbS??e4}`w{+t$Ekh*CFki1TC`H=nQ z>-{#5k4#9hDlmPKdbZEE#~rjaC{{{J%51`<#}!1pM}4hZLUH*xMCNsJ6o{gypLb|J zX_L;8OKbltXtfViI1Y!ypUJTmTeC7Io+y1(vw`zw@z@pG|Ib5#a0^oAMsaJ;Q(GbE z2pqg9See+=W?Lt6QrsfAp>9|AHTV|uT3RVHI^S)%(zAuRIOZ-6`T@OJ*~IPbBYDd# z@G1nArsSBH%HRq$LgQF66gVN87*4v?w~tZ`Mf_c^i-iTqIt_TCuM^I-Pguh~)Pb@r zJ>s9C@w{y%(sW?-oAHhW#Rgzx)|uWA1IHc@)hHz?UskEcQf4(Y9D4|_bS~EJFvF77 zEK>B+1$v;~_%46?=F3{Zw1p7npM2~hFwf=B3oP%jq&~?Vn^8s9pXGP+aj#NWzh1sH zJo8xca!iHvY@y1#(3nx3i^TG>7&!g>h2*2NCr73?hW_guy))3!$abubrWBSB{~#oKE-%^>sIn(UZ=c1C&x7qmroli0;n&-?7EVfof&M@1|Ja z)=H2^AoZvLuKNRm+ZFL^_*OSPPP zC-M-^MMW0zY?8OHS-LeD7* zoRNKdSt>|1FN;A#Vp$A_hJpC#?GM-GCS*&mZZ0pK^Tf4d9oQAEITbzR-48{GlFSHI zO{=`DG}|KkA7Rrerss9HH7mLQA`Z%yf-i$nRr19&zSbK@U+Ybr_Vb>vTSIj1xct|D ziGTNGGykhL?!Lyp=*Mu(e+0;D`qv?Z(_6OJ$6o`NUl00X9q#{l5sSBnfy}R+YyTPh z&*}et(3b$2yODFyRgvMfSszS-tL^=C8=}of{UweZV;KlzumP)HTJCT68W+wJYE8Cb zfed%lJP{n*SHih6jL&mLA>2maJQ8bRIGzlp@T+%nqZ3Mma#%7!X7?R_x*mbWnwxlR-n^TBX>>$X zeGyZLD%B{}N$phTw+wCXzwf{o=6c!`_fa}X=?+D-`xL@bN<)EYWtvKNvRG90n5W*} zcHgjqs~BG9sIfmoe5ZaxreIwd? z9Ma)f@Q$|<>V!Z34&Hk1^OBJ3CuqeMb;^1{n)vwQ_%mn~;H!(VF9w0H#JRb!{+7{i z|0>`I294m><}IIAcR{xvWFCd>edNC+#2#yCg{*4QmJTn2WhqR|2M?AL19--;{e!HG z2723U9ik(NJW~U&nr{lL-iQHvV&fW+f+A%GdCr*gxnwZ?qoCD>QjDIBrpi&>&RAku z`8R6Z_fo&9KZN}`5c zkK#=o_|Onm2zdRZ^xE&;Giivt>N0hiC3D`tU)<*dHuD8IQYHcMYnrY$^z&>LnHb!@ z4~Ua>-17DgPdZ&mS1YY3x@<=|Mt}tk^7aebWCI1-J=wEHx4bbsTx;hTdi0Z`{t59! zAMNr21XO;50O$-?cEq zf*eH8q4ydiMRzRYZw?QCYuld}N6IAFI@z+2(Eu($(xwB7?Bf#Zbh0jc-^?cJStzSV zGln>R+jQ(y(0xl*gtW@J-v82cvB2g@*$qCsDLwl4ZyX|m&bk;$^ni|z9uM=j&KLZi z=_jPtai245E0D+9n-sSN##TGVZ42}?{!%DRAS)M;{X0RyBaHnq!7AqvL?fH}i1%u# zktzyt*?i#}FPD*J6qcsBU7gRHvvHf!qQ(5Jq65<~fwp1>JP5hy-sRm_^z5Wit^wjbLUPJpUgkA3wqB0#f$cJ|nF%o(;@75AYJIn^s6kPf9 z9pg;EqkEQm8y$|nLBHN75E3B@TIebi`>H$JeBYREbg58W@key8l2XGZ` zUZlNu7vhRhRyQS=olnqOJM00jOc_R~e~Ogx^fDXsyvk6FY-qT~VO3!I43EBDzs~)1 zf7&>~d z^t^zvJxY;3*y!K+aD*wLnNn(|LZyS71Vud(!M@P;{XHYz;gE6cxvz7i;^Cb{}u zXnMTh`1F$mW*(o76MHGlSr*Q5Y@aO5#4pAf#t($#BcYvx`aPaE*X%1S;p%&=ubF7qiA}-tXDu$p*HP!L z2hl3%$y5ho2ia$nbg!w3qs)w-fWY@O6JiL4Dgt}?yCTi$>HX$BpehJbtw`833)^Mg zI-}2uCU-h|6!CVg&)^SI;p^)q&+Gn56ysK!-Lb*2PFKS#q-4RTl{LtjpI7+}-7tJx z7EHo*j4kb&sk290a56oj5XJEd1AW%9mZSAXgLxR5KW zbn5ifb~z8QJVN^Ju6|>WYjl%F!aC|Ey;ge2+f}G}x6olW_tVBDh}lMd&IQS=_iIrl zyR9()S(BhbI~+t5ElWu2IoQJi_aGbdXr%J7UUTap|#LM&`S zZ^WmLC7J07iH>Tw03cFmVHS_T8aeDYU>}EVpM zvP2g+Roo#Ce-sY((1^u;1+!?e3rjG!u;ScCu)LXlpJ#_eJJPz|`s_F54GD?18~Y;Z z#?CWJJtn|y&}ZJlyYwUKA!A=0L#~r_6A_Z~^u(NsHXS+68k-+z2rAjaSlhW%*TsV- zJ*`P>&FV<3(i=~>`B%f?>EAOxKhNK7Jk~jyD3Say>9VzC6WW14q+j^va&qQm?Rc#! zmt%&M6TFHy(DS5JF}2B*0C5R4iMJb{@IK&VBP0xTqY*A5hvKFY1Ng69e(fj+I&qe} z+}(o|G8E$OJntIk0vNxy2w$_hUAid!l}VbVP>rizVHlW>2qfL;4@4>2$Pch1 zq^x#**oi!nQ)k;EXu|%X2cu$DX`OLHpXPlH`io=Q)Pdl){Z3Ouis;z$2al3Zv~ttL z0FteN?x_SweYEv7se3hlHJFRO54Q2#RN|k?H0IEB z^mVQA>3nHdJ#)Cc-ULraMy{Q-LKg;9=Pg; zYe=#?n*j?BI<4j&GJWgQ7b!-q5X+*?j>2+pmF(ii_$9O{4y@4)Rf!zuoY_i@F9WU~ zKj9^n@ng@BYwDNC@)Y&aB(z|6=`rO%S1|i2&xO-6L$mevGITRHYoJs097xG*ybU-g zeV37NYQTV)ZE}`aojy-DN6lVn!^x5DyZ(r-Y;C65h=pyb#j!iSqGH0-;(h$TKsqy7 zAR}pH%UVLoT&dYOYGzG5MWpk8l?m9^^Uuzz7hbWU*-kU9b**Zb7oocQD`SR~QaR(M znn+cNtzYH=@BOb$!!{phNeFTT-iCKk#DsmS)N1TRd1bk0v!1nYs`A>^Vu$I|A7t-Ml8wsS$)-moMJ;)m})_&6~d?HVeyjV)oe?d*&Da zH8+WW*M)p4s;m_>Q$8 z2&cQ^6za0NO=qhyc87+&-k3F;!`0~b>bHvZwwC@sMCzbq%mPF95O3+?Y>K8pc2!EQ ze*0zlHB09<@;bE3NLC(j6>598bjw>cc@HT1A7!O>gJWQSSY^C}G2oS06sR0WfeH0G z5|*v|YSQC+;Ms^-hR_%Qr7XXNiuF9{JgN7%Tg_%kboy=gR;gzosqQ1HM_6K?_2}4h z+aa}&wcV~`mS@jF2Dhu(jn(5P{zhOW?54(AbO*$P?rT+B&7T;1@z!F<&j>3D#o9)y z1wjQGNI{JN%-`j}r3|22L@f1x0o zS_dcYB?s8$jzRD1m(bd=-nOvPPD(R(A0OKY%)*@HgK7v!&kO}H&o^eOP+{*!VYsDm zZ@P~-Nb7-F`^+Js4o=tpWtBYMo9HB2o}D0WcS`D1V47Nk*nT|Bb|Py{jVKNsefP(nY#IXrn1X2UrzU}vBc&tnfI#+{?veD&c@Vg$iypgNC!br zCr8T}*#by7yJT9~w)z1i)`?q3@j~;6RN_)iEI<6GSR&UNK;gpDO#`kY@Z6=nu&MYRb)c+}Zf$ z`Rcgu0reOnRzq|$tgfT6g3Ws5bd6uFV|6A+@NsxBPwb3uW9^qOEf-sfW3U#+j|PSyTDZ|vzP*J>obueXG_@zoH0+eS zmYNUL!TpK6<0q&L#$qyJeT zzRR6LrO!v#?q&Wn;3?+@uF{+zNX^*h(2qeRMrPOf9_?t~%}s{R!R(IWwE%OQMz*2^ zwa#oMpMhe;M%ZScqom%y5YYJeK;CL1`$>Xt62g*UQ=$&;i=;#(F0&{z0miphY@QzD zN{}%+gRety;RD5ozMi5h)VUrdbCMC=S^naiP>KwboKOu<`$M#F(o#tB(|B~~1uw?O zx?&C*bQG+zj;F?$`P63mb(^H4`0lOd(a7&Ng|Fc(G=*=^&#&9F+zUwhUBK6ixC@Xb z^yKsHY08Sd=QBj5E}7feQC`cPBbG45J#+org0ub+t|T8;^~)%CCMXL@54%6E->_mX z@5mU((z#-;cjs&E6*XtnGswiZ$Sfbm|8+ny_u>i4*8$((_FZvsP76)tWQIOislQEH zD6@#0o?Bk9 zG5f?268xw`T8xe3%0PgOVRf&XjL{x5E?YI%@^<{ZJ=vdqq%I>NnDFJew$eYl&)j(o zB|M87<9L=*E2aCaKzq$JA|KD^7S?s@jK(!N=#G@H6fd-_NaGCJUlNU(7_$c=$}J8U z+k0sRaW`2$S{hho&>Xf6t=t?iDg>&^l)()J&MH3FYs&?9-wgX?T*P*$aIp5RT2NR| z%k{?XNtAlnpnEzyr5AD_RQ|y3#W7ydOAO?3lbghP=LrD+W?S7UK0KV!;xe*tkqAg+?Wz-G(W4klv-_Zzuqah7!QLsL=z;r zynFS>zt-_imUa0C2_fkI8$c2uhG!P!{z~;YcAP*_35*^96i&F_n?WY~o>bh%#i+Qm z5Zk7(wz*H%=WjfBF_G2_swpPiC5NC5)3Mw1&7~Kd>74{#b5l@-)QjT~$<~>x=pghv zj-GzfD&`H$sxFw-c3(@=w>zj*93FIKP1G|$RfSQ7V5EsIB3bWDS*R zUV;->cOMGSEGfi}2l0g-G6l+2_E2fc=W0$AN$V7OrfZ~b|2PW62No1!>CL*56tmO7 zM2}+t#;|hxr>#XDeop?{Lpklk&-MD=@S6K|8#=#vs-Ka24aikKKaPHMvi+pH+@6^4 z$@*%!Ri73iH7v0kdZWz&7gKxRMCgT~O~=5xRI-nt_^N3^2pDzt-jfCNy+t-#&qEDn zS=auVm;|3y~H_Y^5fq;6RF`rf6i zo+`buyrVj2XO@>i`#kSf4d>eBZDt+y^P#+@B`(7qPn!WcNHa2 zT1G9ikVreyFQDn`AnGi)qAUGYVish|W9ZoF<<3V`S$|{`CVaz*KA$XXZA6r=AI5=@ zD8q-dtaNTt{Fcvh_5G(JwSwBfu}J%X*d&?oYf{==0uE1Imw{%6t{}zY!Fh8E9#-R^ zFnXS8F$#i7{9*7feu^a`7^%vY&IS#5>?Yr$7|&6J>r|5|v|!$WGS~=I_4Hy(O^>(K z)EpWtI(2hx1q{~i=y_dJFP3DYZ zDkFf`W(@z@G9b^@2L9l!PhQB4+J#1fefdg>#~f;Mb%*3;Do`dtVeF_g4iKj^mf35~ zPw=UF8x<>262wnlzeUo^#C~2Xkm9Y~G-N(DPAv0O#s))B5i&jGaM_cw3@M6~X-9hO z9J4tYKjtiv;wF04QLMBTKIbXXA^N`NzUI;o)XNql0REh#B*%%i6F?vT0#sNrmR5Rw2 zN$(J<**OtwOC8LwIe#ouX*IUq_f$G#6$I6&y3|>x?kdvHs9^T4AErabqhubpJATY!-%r8zZ&$(P)=h9F$`t0h#W#ka1KEhIKqxz799Mb3~sK~PPK}07rG6TE?SYlg9P;$HC$06qP z^VDu?iT3vcF6r6s{Kl`0`Z=yHCgeCpol@-`3}rY2t_Q&k*~|hGR@5P!OXMhz2=?`3 zamL4hk7Q&I2{|pv`Sv%+BufytODt`+8nOvBviyt;nk8TLy9%S=4v(UDgJxCo#i>_I z$u+ApLo9yiGEeRrkJh`V_Rq>cY?-gRp)ckKrrnlZGUf@?tkO)ixKtXjOys|YWC6?k zib0s7Nz`*Fs8ve9(syx%M+UJ2_W}@%k;e}O&ZM?gJ$%a75CFzdceyWL#ODmSRa>p! z;{#cgKtbO8!(PVxx0T9MGe%x<58Z{@GS>mP;s;ALBM zz!W8apfK>G37Wr-;=RPKE;wSFEveJZ19HJ8GYae{_BldF66w!Srw|2UT3w^memAlD zW;C{uNx|$%a==X1OoKV|DxS-J`v5AVv72Q4RvhN}~PBb&KR*)35BaN^U9^c0SVlz7pD^(a-i$F5|kv{dgRf~rCF?vL_r(ZK67 zJdw+{8CJ`gk$q;4FD+~pMKhq9Q68ZEwm>29gl|u*StnuSQ56x@Fmzv+%z#{QnZz*0@ex;fcC6u z(1OKveZF4OOA~v!p#i88X<%y1(17=8vRaQ)?)cI#7HGiJJ#L|#0|+f`WuVwn5&ARS znd)yB_%TZygtNL}H9a6nw2sc*_3J(#9KN1qojKD#A#=#4{+8h6i>;{611^ZVE5 z;-eK~!M?l^m|5=l*b?@|0~~OY^)3O;wn{|ATZ(v4F*=Ci(s*GWQ862Y(t5$Y{Crd% zu4nfW!LxzD;PwwR{`iR14v?(V^%KAz2VnUv-#Rkg-Jf?JOG9%k3jArf;=UC!E-CW0 z9*IAG^Aw1BJyOgLPi%c>tCdWBXWo_?e3L6AubiD9Ya?yO4;}{_`d>^bPS?d&9hz;K zfuxDP27g-0_UGFdEdszvYZhE#@qHr|u19WCBR$e@O^{wYy>AwOD9JO%X)s@VRzvU+ zS?G&(*E;YbN-fh_jgz7JRy^=Mk%^us=*#4a)Bffvw;^?|o7dd^tXJc%3CT`0<6yhANqi-yM);fnI0x^MXp9-R#a1ay;S@!7^w` zjZ-K?gk$H;yaO(oY&y$ZXbm@g6g;@7C{m=#!lPKHZ0xa#UKKj>^|{mKHSHT^A{GRGUpw z`wJ#I+A&4xJwrYgp^|M~M}mUnx?#()DUaABPI}1;W+fU;%-^Z|?gR%bD*EX1S7#3! z2J6q0Sstf3ac6jl13*FUxU)>d{rvbUaDFX}r#_%iM`d-2jput|_yY0OnJIXbbcl`@ z>xR%`)GYD^5%gZQ8%iM?`hbuiHoHon_0^%DhjPbv+qx9+I41GJbB`>qdH%#=TJ-}& z2mg{iuz)f2JLVA2q}cI2PyZmyWlYcfAnc+19s&EfCU^$>5V%+ zHA|d*cwW~*5FX+m#tn7>)M;HGwNawIq*_?96Nw+CeGN(!7hV&RcW~#RK$2$y@8$nY zNRx0h%8n&eC3!iGgaOPo;~UY|t1x)8{g5uf+!gDZN*l7azLP09jGA!b!T<1|+Nq6I zI7%p`DApulr5Cz<`>UGz**!s)2%I7UHX)pIOh%i6@AKTM!xc726^%$+qct^Qo}Gz} z)xJ=s2f=9>M>ub`9((@EFWUjlQKC%Y(9VsdCvZ0wHpBJUUt{Tjo^l}GfNLXVM7I0! z;**~*n~p`0?cqUC4n@v#6ul10Mq+mEv7T=r-Lb)HFe)oDvYG@|Ox(>NwIbgi}_{v+{rm=fe7m^bg^*^NIH(+K`mEo$PVKPTO zjz>MB_tSK2IO*^@w(Ap)0cGamYEf$WUNBlkWXI=nVhQl{OYl>;AzQo)vjk{i{8I4* zI+@#!$pm6_+ZzpDnToC-zn4qjA%tU!@ag(xVUf`1HSwQ^yVb7+iDF*8Yy%ko0;224 zjN+6l2CBWd`a~`h8~0i+>~z$OE|sSPag_x{Oy(O|Nm&yEHqiE__)KZSH|CVq2oWwWc1IDW z37Unyyx&bX5_BrmaVLdW+K*NdQ-2&f?wo-~f`DpEVxC-Lz9%HjBMon;Tjr1vZFc=? z?b$!gXJJ{T(vr4cd14cy6(X@P_+K>mDeVE7-{x>Ez`25ihF z1VVXwKMfqln$_rawRL{nW^#Og6FSf7)2FLLWyvljj=dS4bEE^Zy}d~iADhuC6^RC3 ze2jbYxMchxPD4U=f6DK;B#GrqbuF65- zXM!)}xZ{yTno*hgqbr(RMiDPxjGc&*%p*6IGWTM+`WBcch4p|5j-Se*t)6pvu$iKn zt*+`T%IDtNisqI(*^SSO-6Vo9C*VE9EP~HAcbcr*?q_NgOL-SGjphl+oepA`Bi9?a zPnYiW(bM^ZqA@UiokS`NQu+BatZIKY*!WT@gW&((;1ZM?j ztv`F-J^kE(_-$bviJLPS&PZ0EHi|(rq(WZ@>ms=l$|~FjJhE;NRWk1)B;6&%qF-)a zSI8+RE}{HuJ&S!BV-JXYAkV4mm_?XHt>Qb=+_a&+QSuRu&*tpK`vWXM5i)GId3O+= z$=dgkwGcgVaek`*@$1pf!`XLGzXKL^=#~`*-QZJ&mTl-p8X|txKb6P%thU#hd^9DQ zR`JM0YLOf*IK0W|nXFW`es_p_@ErkIUW^+O4A!J?2IE4X5+zqVx>RB6tl9nx+`@Kar#xEqzi~ z5K#Vzgn5{`^RLf|FjctXUE~8WQjA+r?PY9w=#K3aS(>_>8Meb<09;B_LdsQT0)?sq zg*cc!Lp;k8AN&~3nR`?5!-PXa$lchUs!cng)h%*uK1-< z7HsxRmajc}sF_-@Wkjs%A>&>Wqq}n<%8+ZKtu-f|M2c}e=ViPXQHNP?vWRaCUKBGI zfzIkh#;^+lQVb7}zVHG-NXVX65JR|a7Bk`RG>s_yb(uCrZyqoL^-GT;wQQFBJB4Lk&5`>)ljyB^P(y3p=?Z~m$3U0S5UHs6+n+ZV z&B0vCLxOg=*i7T-@_P1>Wdf+spf=-P*R;SiBpt?|U=}7Xk4#EtMmBws*LobS#afO1 z;ZO2rexHu`^12tbn>f6%4zEE(@`haWK}L`^FLapHPYGtvFQ>^AIP8Eb2S|fre6qa; z3hqWG2?ui(zvxg!=4V+I8ZNpYplN zA%UGcv)enAdB#xRj`XsfGT1QPdd{!XnK7i#9a%O*x(_J`DNMnSBoQ~=H~Hegz;Lgd zK!}(HFbdv~qZ%zN_ZpZA(@jMrfQb%ija$rKo7d8veEz3VYN4-b4G%7G#sq&FopZUY z4?i=p)z9_2JjDqNw)BA5Fb(8wT!|oQ;+Hna3 zsIP*T2LL}mC3U&HI9#4iMWBn6VZ))4`X;2EB@YG(1VUQVDbBM4$rsQ-hq^`Q1YaVw zq;kBX-vl1qO2#{V4s%kqhYw93H(?CZRw<>}?39Qge3`82p%v4VHat-vpGbveD5iRa zR?~rWTCDXpwT{?-griD`FiO|DEOciz70b<^?O-d`j>Spp(*vUy4+~6&UNN!a31;DC z>FMY(@V5zjzP)8LBppRhGKX2%`x)R(0Nyr5suR+X2BM@fl3u{T>ZW45GF7*kGkd`i zoQ;S(6ITN7$_%lZfYD)>T_$x;71+=FCzGx=E!OwNeG8;3%S)#h@wfc9<6Fb)2%-G% z-|}BylN~iR3|k+8xnCd81ygo6`U1Hh*+%nQVCKgJSKPFJbC_lFv-kT0==F$c+2iY@8xTZc}>u>#&z1+wqrO4?7k;P2({e zXdQD2+3Z>R<3P2!f%zYkjf&76 zBih{g!@G|<6Li{;>|kBPJG;FYL2@oCN+^tYy(M93+BM__NQk%FlcChiSP9GRsJXoN z<(RfBM#k6Q7dQv93dIm)HTvLq|0<(2MGyg}9ejQRhtdqk*Eev?!RHjGYMHXw^Qh-gf<4vn(ONsozzs*iV4c(IHxFU5ghi}#rbBL!o%0d-n_cU7WNF;CAE%pfa$X!UM$~aT zQ%uGcJ!$-_wG8PF1y_WXl_N#sRymcs8%imUZQ8#FwzMV6YG}aYpx1)0h9y(9flJ8J zgt_U4C_vw=(;{2)vicvSRrYw}^p1Q41yeVB7=AlMA%U-{!`SsWsmZ9{oiQo zsQ4{oFL)nILP8@0^17NDdDRMc+(S}><75&b-rB3-jeLS42%gziiN!AEa~{jukNBmt z-*{nlC*ty+P1SKIo3U@IN8 zPqSy)92*0tR=9P1g4S=6dE{p~Hv7eprB>C#pz=feqn0dm!+5#pUwV49qW70%xpH^u zR$LUXa5+-^PW*{p;uNpe7cW1DglPg*eq2<(xvC>Oo!yjjl_pnNjk=>DpqQH6A>nFe zS^F@)g64-*C(ao7nLsi^SVy%B$u2oIFoh<=`gwcRyGI<^Jo3Oh)hkda5tMS;84@~U zfat5U6sk*~Gy`v2Oc#*P<-YXfQW}u4hBAIW3GHrd7N1`D9ahCqZ45R~DV{DTHG#9e zr;rhjhBF_%63-MyX6PA&Rai6#GJ@T=)?~W1;arY+KNL(4%3+JM*z$$637|-9FC6!@ zgVo;t463)qVAd8P(NE1;4y!4}hryVhBJFI^qwQ5&1uZlP#tHjzH1E?O_uHh5ODdVj>5}%?|61jP0?%2LtD-+b6+^N**v;~(m z)RG+ir{g^*6X*I@N6?HXE82*&cB~x*{M-xTHlSoJiuK+{ke)zaa?6J^M~bvT{D6c2 znOu8d!IZACS$-#l&VojKSA)=mF)go&vZx;#=E4;WoShhbZ8}Kh1m_}1g)_bsx*REI z8_Ommuj!|v&ubE3WcX=??H;?1BUAV<`uq&G6)A)4(Rs4k{{Q-8sEtoxReU$HGi#`g zE*A^@JTaU?R=;P~-lvbDt1d^19SwBxpk&z)YzAgMdB;N#Sb)&)>X=y=tJpi-)3IE* zYc&wZCRptdFa48Kxem<`_?16;Jh@606XywK$_)B0((~LGmkFik zPUJvwHV+$g!0IfoHozB0YZ~_t)>ueA#}5_ex+d=Uij-}I`Dj&AuzB^`Icm^&4=9b>)vLC zb0am^|jGt51@VFSqjF3!HyfZhwuiIXbZT4ps1o_>5N!GW@Gtez7Qx_h)z% zUPIUQawgA0yX}Pqzg6#C>p~G5x(dX8-)tEZHz_C6j-O(2fwkoS%ATp9c||dZ3q98& zCz%#A1m}`cQ|&Zop0(k1Y2pvb)d_3l2&?1YO8#g^Ya+eQQ;)=&-Lin7Q!8vW(=e1t zEf1HXv;$tMn_5vTf_0>JEI}l(jvZWs{O< ztD$T+mgUALJ1@bausK^^ILlc3O;76-lT4MWvpr%bfL97-th+Ti(ediVe;vla5Imyg zfF(yvlMuNeTHHtr5dCU4mKq!j|FQ#!a+U4ihi@Jpbz}*$RAL5C{9a9Ft)T zpHa!-IG_evT@xO?Y^G%&9j_KP{&vuC^GwaeVYt6a^`f`er` zwjd<70jpk=T)pT-JQu7BfPT_U+iQvQn{SKk&C(gcF%@c1DdBI!ev`UR$_fxg#J4@E zva*5|TCp5L0pc}Who8`tYPiZX$+m_{W@P)$u|V~bZ;=nKmz@2TcsHl9ss)^a=dJC9 zRNyf}PBkWR^EIpKBH?;BYU;+N-v?fIW{}3Xb|;;~a&5lojC#RxT`9lYsGj8PfFXB_ zZmJkmgL1|LXi!%648rS@w02fUUEII8DnuQ7bMgKT|EXW4a6 zFevK2Bt-BNyjRdr*EMYwd!AmRW!Er+!f6T&fRyUs6VAFaPSR3l$B*+~o)Aw&Vny`QE z#fyJR3XN^yy_njB9KJQyUOJAlPIf(T*i0+dXMrfSce{=g@VeNnyo$Enzb})0&nm!y z5ExvzEq=2-$;8)^;<-B^&G6U{M5^$ZlBk#X`-lOTz|;3vLfUr3f?K47{#$1`wrH+0 zA#gh4&uc9-wf^1U6q;oc1%9;i#C_BFq6-Wu9qnBGQ1^-PZ=4rsaFq}?@X-dbpgJWP zUzVLo_(F3(6QrZF(NtZq=#ad%K?q-o_CoEheeMv>MO$Kum22~;W<)w_pNro0;OG0o zzg}O5nscK{iTdHFSXQGZiXJhmF!tqV^?@GBL(1G zLk+_utn)ZI%Z59>;q4UUV00+kSaPS$PI{4o9@IghZ?l3>*b%G*kpPgNoM{bec#N#@ z-%4w)jlo(%5$Q8=q)Z$~mt1;;*vs4jZ`2u#=HX0r$z z?x&QB8bp+dJdC{&Wgfzuu#I$5-|Yq^xq{Sc*9fk)W9`&fPOkiRoAXYO{eD&-iSj5#i&18EYV{?iPvr=b{bp0 zE9WnCVzb12B_~ofb#cgmDY#>O5Ai3T95v)}hMJm-G|BI|l*buGw_XhVoAPOMQ>X>j zyw}B#lSivTM1LkXjrLv{KoJNjCri5_EgO+FxxL0HS>dLDpgpGk^thZ6Y{nq7#629N znkmX*3CSJbKqo@tOFU-`E3=py2=w+5vQ^KHQHA`@T_~|;AMk2#n;$J0Unsjl+!T~T zJ@(p#lfIc_+TedjRFJ0a`*dk_zk3>x)p9l@TUD_c$jxjDk4n3Pc&+J7Ii5LVxzcQ3 zT%Y88v+%NOhJ(2nW(8DD^BG<+5W)f9Ud_wmSjk-DxofD4nX+wu;A#u+j%E}vwRZe# z3S$(GJE+5(gtn~a3GvTG4taQ4q~*)t0|?AJAsq6=l{P5p5{BxzmnLv2675-9Fe&fS z6ps(<#*aUy3DtzH$763%0Q9oZMV7z#BYqu^2cziW*2zxGX802v9=L&cqka6rE zN~M{s#}B3Q^oLS8$1+-E_8+D4zoq?C>`ec?+0Dk4<>B&7u)Z`*I2d4q?`9tMHZd37FZ`-z>2QHxfmA& z4}MwL!vS>|;|18U`A~KiGQcugCeHu1qny+XNc05ZnQXK{YKb+ zwFD?5Obegf0O9d5R$z`*mGyLluqGNNry*iW_oZH1wFjvxS8O)7+Xd0`GFCwN?_%sq znI!?gD$Gq@o)xGn-?q9wV5gH&kMnBVULyw}Dzz-6kKMgE7}m#dzI_>p8|}Pbr1ASS zyDakTCqZb&5;gI~PMI@2H(slP7@n#q32;`ypc_?Xf?ens<`xfS#x|;2<1R~cc#Hda z^&F??8-B*1$Iy_#_mj5>NJ6I`f@JO-q^BKoPMoWeqe)x&rYYp@F?v-yHC&z=9CtAF zdPsMR&>b!P{(}S*vdji(KcqVxz)}{minLqLn8rV`>7W;pBK%F)eH70sH^?n#`wNct zklfsx=O9&1xQk3ef?J=+tnp@SZ}>L=^f@*bHxHYSfVOJ@>n?!gHr#ek1$o$&E)<#H zdKyGqeDA<{%-6`yeTvVy+q~}G|GjI;BW@4ARm&V^;oQ#zPa<~Cl ze1FF9M%r-xxdYhHpOx+ll{Eb{?$2DQXP_yuKa5kwB3^S77nwSd)AoJm6k8K&u+PYR zNjLW7&Xue*MA%KI-_fs`0U!zxbmsGe4!USrC_`WIgA`|k1il!8BZOJkO-_Z}y_FsG z$7-EEmiMWq=5)GWO11B2V1D{L85%QIgx@#!oxyTEudRNBIayza4`%nV0$DjZSs$O7 zuKGVV7_X25pRdk&u)64VKI|L5WAF?ItM4JOK){94ip~9VJBuoTRH{v!@9EQo?MAi5ss5ZvkD65 zqn(*Axgx8!I1HF2p%^9*_s?VFOyd88KXoicTtRun2?=4zuOfX~wL@J_uAvI6gn9BH ziAz_9Kh~zA)G_VnVG*X-kcA@Rl$tJSt5IPn+^4?sV1N5b`@OncflMI+)S9Y6>_T+KOxKErJSsuVAM8PHT7*os7dOSl<54V1&l7wl;vJKNlOJQjwetj+TQZ zvsHk0A+kJVX&Hw+j4r39gY0*sF|81)Fshi`Ryga2r9MtGzfBhk6c?>f^;+B9{0W4i2)$craG;m=>`-`nc{bnh* zX3WYWrQD(J%;ZwM?wCL>>Tk_gtW4IZ>qY&0%zJV(;Je5()ZyQ;ar?+JG)=IXsy4&V2K| zf7J}|+K+kuIkFA==XxQS+M0#OESb~bWEbe6B?u!R3WY4Yrpn+ z{WmWukwM2kLGSX{Jp4dAr$jA+O@inJ%s`u*iH=H#=sqe32|Gj0rU<44NB!c}> z4!|2lZjkgs8DT?ys?hW5Bm)}p0>9JBVc0US0{o{C1h}`?uxP+T zF2W~|&XMBa{Knr6CHg!nAwDDRMnn0ys+FbmK2%%#=Sq0Ezah|@a({JrokE%6iAc}w z*{k|M(JdsTjWj&xy=7WQrGU{%q*?6jYq^pl!x(Dg#XV}}KvH0rp%{3CXC#sUwMDT2e|z3D z$rJZ0=nDg$nVCngVC-4N3Nj`(*U7##Vx@V(sh(=4$7B|?NuM9xLR;WXrs#pT)~Vz- z_PA2>jxrvhx+2%%(XN>OT|4G--Io!q?OTbQNvg^bC?kj6g7o4hOIo=^;#+cWJ>9$SyF8TE;FyE3`|7W?a(odcX6Dg60^8l z^#Zw;3fUp;8HEMjFfvN zOVwW#(Ss2GYXn0{*#2>)q*0vrfhbfz`l|7sN33I{o}vM5n=cp$&Zbg&PQiZ_U&*Qu zNnVr=hPd4`Zd&+jE~4|z{MB7pTDy92i4E2rb#G0q(`hM&VMdGEu~TXK$-?6h3A+xV z0;g4pTimBro5wx%Tb5Gnkn(l$5$)paR`rm2inI&(2O8WFaVS4O99l@=90wa;UbnR?{h@>YyLX_y#al(d;L3i@duaO zz&+3R->Kc5L!&kyhf0;~6hQc(N5-!=Fc*5xi4-oiX`Obg#D|CM^hkaB(gXxvH<|wcOFTYT%5JE>#lCxK*oec<4Sk8aN_Hv&^qS|hw+<%=A}09#km~>mmkY$I@Esls zgELFD8i)=f2>F-``r?Y$__G*{gTwYpKo<|{VXl%0oK~QyAo_WnH-sR9{+MfiQlJPs zAI#+_j2&&s0`fGf3pBoI0tPp?59fr4VWHCC168b%R@9|7GULV9BQkZEW?*?gm^puQ z+sAO(gWLObDnN?fQg;vE&;FPjdg4*&YESo1xzTnRI1r|nUyL0GC=@5iKf;{I)hu#3tM;S@ADyRFlDT4>3#mhzH=T!HOrK9LSo_>`jB!eo z7bxZqrA;hyl33!s=wlUPHH|t&6=$*;qQByc%V51j<_(5)?_boFCf5&?IPkaMZpgSN z)=CU-lQX);l9uke4_UP9>Z8a(CpLdMlaw2@y8i1;5Z6QxNo~Y5l^z36X@#i4j)W;z zF7hpQn17@>A~~eR=X0@W+y0ZrNjOV$G*YR(MVG6&G_p_i@||J=92$0^8I zFZyPU04;z$M$`56Ckq${yjEP=l}+f28A_4YyT~* z#vbO81`ZJQjh;}(RJ6fW-UW))z8?=d_-WOj2dHQE2$}xyF|=&C(zz^%AbFZK|VAOLw3+G)PepeGFty&|a5V8Ulr7#4uaS z?OqO76dw=4hy;0$6bTs{iNS!=u;K1W*Tg-dkTA)f0GAI2bi%Yrpl1ud;8VT5MjX+3 ziXHx<8Z@@06&0LbZ05B#9^hoop7btc>e@=j$&BtRDR0-TkJ==ujyKyipKW{;vTMvV z9pcKNBf=GM;vWpbnxyw+odQ`Fx)!eLi?Jk!-mJ}T*=TilYzLQX7ILq*gR}ukX>>GJ zAQe_dApxC(pa(^#)l_wKnSi2C?WCawGT!S3QHM?c%T!e(Ji;MpU7-fQMQi2g);JT= zE~0T}$sFU6YN>%=MoxQto>6*H?o9S^_qT|RF|tmzl#V4&8IEYeH{-a)2XCIA0j1085#7qLudG71R@!V02%oIgu)sq z?&$jyaEB~N?$qq$Jp)&P+4Iy;hV-vNw!@6!0b>D(<0@yAUWjV5 zfYkdkcY=Ey{j`T<`N6ywVY#5Fu?<%$`4lSSIFbE29VI1+-6ud@4}ltBX3-%RRPo|e z^KQMtnl7>%l~T>u|H4{=(7u!(02$Tp@iLN8zZ?fH@aL<+K4@ z1t-`O6Sjyo`*wQ?+n27kNKsG<0)$$;HXKB%ekYb>4Gxs7EkBkdgAhW(aezRK^>sBN zWMwW)(LfRsh&Xt%Z(k^G5BuBgrm(J8Vv<=DFX7jTqBVvmD_{r;eNCsVKirh&;rVfAj-8+ya?NShrDoc z#tTl#0&~uS_%vNIFfkKD`$0k(4mic0niF<>B+-Fd;n^yLK16t77_pWYqLcjiS`kP= zgy_I+Y*02dSQb_oqZ6Rf?ir<}cmSsb&-5I21OHAAa!Q$l} zx6x}BxUEd+Ex7h2ndhB+AE$^*C03NA+hW|4?d!f=HWP(Shw#&6zil*7C&vM;+xW4@ zZH2lVAvLu$WC|E$B@Efir2O)kIB+xPE5g;12H!=+GIkYTD@DUM4UQ?UFr9~Ih#kb^ ze3Zcd_P_0MkUKM^DW@_{jzMTNgkx`%WFfZ=R{1wzPZ*QM6Qe-D%%EXRwIqL+`xk~k z$n~dwDWh#7+Wv(Z^6umTG^&qfiMM+RmkB6u$hgr-E%#uOI; z+e3qT+4{}Xplh1N(4e~t@nZzK^1zgdLZ(C}^P9nYkY@#`PojSSLB4jjQVq46{6rT3 zb;B3g#aZ4EtR%^)HG?UKYskhdaX^t3`-lJEre_rD96fX^xTgOAHV5T&KP zNFKL#WTg#40d8!O7$*-CU(tlnxFSE#P5U{YR~=k%?*?H+v6Zdq%Eyz*XUU~nxp~P> zI-jP{Wdqmft~w_SvH;sz-MA`D$?zx;6f) z+Pm7HxR+17a{aUV@n|UD71BvqH6uJ28r!k&X*z~Z3N8vzj|-(F!HkTg6ll|uqdpeo z%qEMvG?SK_)n$poWQQAX!hW+Isf(LJ+!c-N0@|-Ahd6DB!PxRJTsJN58lh0zX8{|8 zVj}dq2zyg33PR>xwE@-uyf)yOm0KllEVE#=wXj%0Ls>df-K;mAi|=2B>|Zch1D*yp zWG&#|g1~A;f=Ve8PQ%e~uvcdD!Q`zS``LCwtt{fkgavk|u=URysc!693%@k^P{-A} zq(z-kpLn@<%Rhl$8Wn^(E&z2@@abn8c>18A(}2<`NmbJ1Mt02VL0!(YqV3$@ztZ+Y zb%{9yUE)QaRL)7|d`y+I&1>0;Qk_4@tj~j?1xrRUTCPogjAg8BSdLEE)2awFlsI%2 zY?-1#)I(9*9Ik)WqM|b$PFRMC7P=^LpZ2e-Vls*kfq6dm!g8q>@q9JZrY&2IZkuLG zs@FXLKZgmEgl0*N(Sz9U*tNB?m=MB}OU##I>C7<;C4U53)h%eq6OGGIh;~%%o_3_f zAoFTaRP@O*u1qrv4BMfZlIUO%G0yQy2fiRsV@p=O5@^UH(#WTA5-3Vvp%XS*)KqsQ z(CA1}Q*{}ugGY;+MwkrmRjjt`qs8R1qDmAx_OEM{PqVXHgF!CsV2tNC=xx)9`by@kD>U}O zD&K}`CXPw>$f&F?O0i^DJO;m_H`UJ94eW;c`3`VgmrOj5vw5`yg1z^Ix7@g#k*M-rEu!Tl)R|KTqQuFIT3@Efq{M&>z6Yr z>3aO4>F>-+ZR(sB^ODgN>M7{IMy(q!QUKX6*1emg_QhJunB^KZ^^#HPs{`biIevx_ z@raU-74O;t5>eBp>ix2sk(?$QUT8jo<^i>YIwBIpu62aas}}&oX&4|N)SEMTEccc( zwi-9Dv^FZqQHLNO>8c*u6m4_DsD|Af#PEnm-||q1(t*vw%VW=xpZ#3zwvpFIox|JK zW~5k;zv)F}+as1BHbZ9~7xRGvYgAN9blO8GK1oYTgF%@iL;FHa^@Ov)JnaOC;c#RV zQ3fW8VE!>d&r@B1H3UrrU?1tXj*aX1@6culU*Pbtju&BY*w+AuSHS3Y5$7Oyp7&#t zM{NDO{eb44k`Wj5gz&s#Wkq4wO}8c-3pY2yM7F=@!>wvEtW?2#TZIYnC?);@anFyH zSs%*>`YB4Vq1@g`b2EqcDT2ycJ9*xug?J)}BAT2xHFM4XwTxvvhR!UBh{l_Oka$s` zJVFhj%5{;ms*}Yw1KWN?qg3{g5r?mNlf)1~>NR}S)0JMs^Q|R3nuH_E|Ixm$xVbslt8`+nthwG zD9W^=JhKZ#WxK~MXI#;8%?=ePNT2!yTGLc1X2YW82`!%lJr2Y|icCb2@a)$pHxR*C zBSpsThRQMW9=zY@^TzUqe{>1uzeHw+_XW=%E4toa1a}fgEg3mJ2G1JL>6#rM157@n zDW*5;Uk>e1Mc0P&S?PR|fIn9sRrnkZ@4zkfEEL`wfJF+-=Bji+alX(lFJLLWV5L#JhVP97GM5x@eGJo!S04EN5)RX zi|XIMbS%%Xv!L40LJWu%-LML~DqFXHoiJb;=;$C<&>oRIWeFX$aT%wNWQlPsv->S8 zIm8iUkRF@UopClvzApHRCsa`)@GObJ7~E8!Mb%^>DxNfARLL?%<&fvcFEO_6o3nE5 zki5DOMV$`}-ne8HShw2d2rbuu_Ua;6# zV67=cb4Uq@F0iFLXkAMjj#fp9es!$ig+_fHN2oi~ryA-GCz=uhTntXZ(XKjT8BZ#5 z!WQd={ed2{(lxS@(R61e;f5eRcqLi3f>I?A3<32yFaWq{L@U$LOiN?P(73PABV2Nn zCJ;9Yq43S!&%}D|QlY$Fi|*Mn3(%vBy)&12z=C4*qYymyJ}|Z(N4svYWk*OL7b%zE zvfa%~mI<29p)C0=&5nGmFXzQuR;;VdOY+?FS`>c-t621unq}5i(_u1SJd;qIvjOS} zik)g1=UBF&xvY4W=W|n~uafCIy6HosSs+tB3GBM|B9VT zb+gji4gS8|S&SZOxB(bw>SMfsGEi8B7=uh?RWj;Q$t}Ub;8jA<%nxTD-k-n!`Xza# zxTi_+fMJqU&&Ne{5tWiAsEBS^2~veATgz4}o^VzqO-KnVDqb>cC{K~-JKC2Og+YL^ zU?c7T=s{5fnn1b3YD03CixN@u8fFC}5^j{C^9rM65|QUvo3|3=hOsSLByoWo$b!7i zD!$EN2cN%q=DH3%XOiqhU6M=`Yo(7PJyX1+tUvoi{cPeTPIqnW#Oa;&b87v(v3}02pQZhKdHR>r;Sh_!y;`&C6-a#Nddd-Z{BPop zh*iZ3EYPgWwS!*Sp4*NbIlQ}{{hD0yoQZlsv)EgbcN*9keaeE|QQqXp!TL10;mrzp zgrnc;x|Q4HEhxcnMG|$Aj?lbaqL-ER)C&-I7@%ok_^A-vB~5Nz0-!pTI=QNu`A7>#UvXH5h2h+7*9> z3%3*^V}p5SYUM6jdCSVHOyb##S-n^KtZ3Ek>g}Zx9%;@hR`zBI;maaFX9e#J9(?E-VTPFPvt8$j`|Z`cn0X;-FeR0|i?49c_#jfA9l zrP^76#a0&z=hY5_e&L?pvXZYL_Q|_c&}Z9}2OTfYUxC;C9Su}L>Q2RcW0XCJuAmhK zJH2F)8DZL#@mOA$D7f0LNk+ryRzjoFK5E&b14;oM+bWMKEi)R%Ln%bJoZUUuWpCz6 z_?hdvWKSV0;_1J|B$cS=>f6`fg6Tnr<<^p|>P+9E1)*F1J?h{ysGJgmNL%ytFOO;p zH`bw%t9_{}klbyUu5HDvqb*9M(ML>2ogf1P8;i3ZqQ~tMUA88GOZw^28Cg;?hp1wn z?()M20TS~05D8P)6-V8W42Zy65b`b#k$1@&mzZwphV3S@v-kYY$d0}XNOK}FhJW`o zC!%?j5hEqE3tDXq5ap{gkg|-MxT{y9ecNYG8%e_=4xJiJtfZL%qJs0`Qu+G|9`Ug?4DkiHYglk|P^p^4r1T>xY_kezMH6km80YI{W1@WAb67mc%(L;myd8;~g*Y#Lh@i0oX26B4bL>?Mc2!!~O z6?I0v!(ezpK794+i!Z+TlR3FiP6GaB0yuTm+FNiN__N5>ce4g!asE72&Ar%s`N!NK zO*W0z1=lGBxttN{GO*+`vYC-|M($?BA?pYB422X^%r82Y$=<-dE$AktRD#soOi-5~ zYRB$Y!lYZ9z-+}(1~do5QpD4WZ}q=SMV3XLK6Q&I0~1!7vy;;sOz4ASp1k_tp}L?y zd2@^&7@^Df^}o+m_j_c$FJhg-`rd)Mo`1JPdR|z^^XqqR-7cutjn(NKm-CS+(uzBg z!W&}sOOvH5s}SQLB^p?o!^BO<)QXRl{$Ras;1=>u{ItoI_a_`^RTU(5h__afk)?_# zGN&Yg-&PQH=|?Dc3Oy}Sr^Xn-GG=mYycZqL!+`zupxz=ZOLHJwtR~KSr=!>~Yb%uuuRo zuPdcPHpn3mdhMm#l2ry64Cg65c&I^0#ond07Pu{9C;|MA6;-)|glCZ|9<=DZ?e+D> zdU|8Lp!?a2!0OtpD+7*Tk2%mLAzP6y2F{pm;<^LKomR37w%Gz&tKr=ksFJ1>T5CYy z06|V@u*s;iP2x?>F+M|Y(jkd8)5aipb=bZl8o>Jj2j62s_Kt49NgPN{w~lX+mbyC} z3kVzp1C9p*js^k#ECGNI2LJ793U86u;V+l0D_PK7)CE`t5|M9fZ*?0B^%{#`kJdLj z;^EVr7Ie*;QU`M066mKP{lI&946C2c>9(ztD zO?a_B%yOwe?wbcXCVRg=JIWBCyNfUcpf+P8@fZc9kI188b?R**avvTAIH=%lvMd>e z>eqJ^i+gm|aL^d)GI&iUQvFOO(#b^n%a};_F6$!^Ju61;_e1$YR#z8qv-jXIE36Jf zh}pdF_aYWB>>_2`OzfCu$4x*&EuD8BNVVFoGH-lyYgVWa_d-LU+VHB*E?GtX&^;`U z*3}V`O$3OjYyq?rGC`%RbA1{yJj2mq6SQG=F%Y!ThG*#Iby1kb} zzqejD&~$rHeilZZ{l07Ut>C6k^R4vN03dF+4LA=Fen*p?W^`QeG2ZC8@vVd`S@EE1 zR&SR2MG5F!x^_3!ZvjG)#~Mh(2DTVeCUkv4t4*}w6(Xx~a0Z`I$DvUJmlug}+{P-v zDAYAau}0l2-7GnN>8->QrP1RgOR=SEIC{}@?SDv8hV{gm1W2h|AMZdlu&r*v)Mqg< znGjFWE?M@QrY-#nJ2 z6rwtP@b>Cyzq#L*(Nl-=eu`skLu=rjb8me-Yw%aLp|?yD$#b63G5~vB337?EBFnlW zthnW+C^U3r$w5r8oM8X$%gX(@vB_{ zT4&}1Az*b?(=4zC>^{kA$!{Ne|HIIg4Oeu&@u|QqeE``aGlf(G_{p{|bRklCY=5gL zSBsp#*N{`1*)vjXMT+9~Rns`=HdRHF4T@g6TR^~fD36pEm9Ffh=zXKX0czr(Z`f{z z(5M(~{~y*Nh4)>%+z7sZHb4Ex(5VTDNZ4zXaQ8;116ai#n4X@_6fl6B3-Tu4ddgyB zi7RA(G|M`TNX_vr#Y4M=lLF@U2dl z^iRNq@tP4}1TU@+% zI9p6;(U6|@BuEq?x3sKyQfIUz>YlDe>9mW6v!I$|ddU(|q;YECRsXAgF^4THxkydG zxclUHFUpC=SJ!7&tPI_0E_IBTtm+2+K;7o_)S~3gja`tx?u@I>lV3zS<Mc$Yg_F=yQje;ow9ph54n7c#tSKrryl?TTU8fA1)0R_djh=RFG%m-;56DXX7O z5F(l8V3B{S^G8X^)^Nx0_8fS_B5H{YiBXlrAwAwg4aisaiqn};7$?Ap)5FbG^Zp}ZMzSMl}3_kGg@Yk z#^hnTR@$}S?))DmRDWBTI6coRmm6BTNPm!EcP0_FqMc1JrW8=od670Us5&lk0m8P2 zveMS5!6lAGd-X&Kd^FK!hDx-SCNEEJE5xV)XbUDUNg_%tfbIgf@S>k!N=qNbaXZB-N4>aT}qbkj=qxN05AK60Vdqa}aE zt%7flLe=pL&yHz{NHIEMVo(}DGC28qw+6-P^I}Hci|V|HK_9YY@YjOL_o9N|JnGWn zU`H;TmdJGYx??e=`*Nj&I=TD zCq&7iYLAK%M4)IDJfT@qDbc$B`9R2+l3XF;GLko{7jehEW$8(<381~)6H z4N?noMv6Mi6#r*lV>)OUXm5h6t_`(oMHs5?#xFV{TAqnlg9&(Q=-nkeFD^uS)lre% z`)+05>H5Zy(Z^D9Vt!aQ_O?@J9$kR7M=+W&BXngdc}dm%cy3J>+t|`G>1&ycJDDZ~QtuwKtnaSF4^QNo4cbk#x zf3_Bc&H#LO6c{^SL(y@xD5z`r6Dx(T zKZ$~oB&%iR%Ox?#2#UXgOW`?63b4XJ_$xzKJA}Y(7DFo_mE^_K7f+wRR2m5Sgs9ZG zW&5lBd_jJ8wy0+Eocs*`f1&^X1^Jo$OvnYfAol;&|H#i|t`=EO&9|D*XI(;SW$Y5E z&p5DI>=H;$B0TN}HRTuo!+Z;M{wpF1-NN$|n8GQ!1IuAfZ_p}zmQ*#()L>7(m_K(% zBZ#8g%qCa|W5$hTD?>y9d9NBp;8u^y1?Yc&ZwOMu%8xu@R$Yoo$+Hp-X9klvJc`#P9)B zxwAgKbzuLhX(kovUxSma+*=X9lMxSY7fewU7JXTeGd%bbJdA1{URA`i&(1wFG{H^t z??7-h?~znO~7}q^x5Y4ox3-xxv~Y%W+vcDdrrtw$AwYwFxmG)FS+NK zDqe$z#6}Tj=9 zWJU^h2V7Atj`uzdESgS=wVCNm$qi7?U1totiJnv&Yc#npPQD;>^6XTd@&d0BY@#ii zC>HHSicteqN`cxD3F$M?d{21jDt49v=Vp?7ie0lO14>;-dGlS*LzTM6;mL(a+1YBv z3tsImM8=c2NUfvyd*4|+dL!=09m8dZqvxFe1$fl$K|EFeXwDY!sfN{QsnxKYkVr0J zo4?&+h$Ai8OEM>a6NM;WqSULpk}WH94#R7i;zg3xDNBudW-(@|#M5)IPP{y{^~Sc5 zWqQ2fPCvSNBKR^IMr*Sh;4vHxsNJ9|z(Q8b<`JmX30U;*(_CYL>Wg6Nja!G@cZt;r zJ1-!*R}^@}eAjH1QCCh{goQ(`>=lG?jJ*W*SH3F%UAXB0hL3oU`{1$m9UkkrYS|-UR&oU`y zirkon_lTYCu`hOp&olr|MgtFTL+Q#!n0P{`3j!g0RIJTU) zbLf5Evrokt)A!Z3F6V++Gpg^Y+v}m|K3@Skk6^MmK5nd6cpdxkkxCru=!MRfPmK6W zDwbs$G25u+png`O)xSBk+_lWP_hXP^GBymQTwVS%F`K@AvQ3gIVhHhB4 zB_&I1XJDgNxPaTA+13CAiPV&l9CbGG2w|1nm%PKY`3mTpNnT{PtlW{UkP=7-ka#L3 zUqhx>z&%MT)F5GYM%g;j+SUyBHR8*|{`-zqOVn(veLfwvBgh4-UJb$P{`|9$c1!}+^6@2}3@ zz9d)nr)>!o+2E`2!iq5;;%g7XJ28M20}bzEo$T+aKgJb{r(LOa8$1m|6_kyGhuv>Q zSv62IOfJY}y;Z*prQs<_7wVQ=6H#Jalk4%7GE@VOKdBanwE_RxPDE}(vtrhxeH#O~ z6j^Onz-kM+bq1gTsEeX(vb%cf$Xu^~;@bn(Mtye30+`xQ5wm}zU&TNn?UBcW0GTzH zP4dLH6t_Hesx-VXMK}JeWZ?gJ`De8XgNdqOZkMKwVY)4B;5=|LK13{U;)w|^~v+Ue-G{jNKhg{P?G1RA%93bA_)Q)z`Ymu?c)iI z0)f7~xjxtX2Ee*(sZk#$NN2pA1-cBF?W+Ze5#2JjQVp1a1!jWyIo+1#kQs}*kuLjc zHYlwJ@of20(x|&kiKifHNGa5SBB&eClQt<)iaDM0gcoZepytgd=r%;P^hbpW^Fc87(Cg=b&f`)Bk3ZRXF7M-;y@V(Cq zXP`(_aEZ&?S2F{oZqdBKnyQ&G_^DASw-3J`H=v^^>Q*n#9h6&I7Aqn3-T>AH-9ZFK z-5~^V>}-{|fVq34vqlzSen5g#jY{uFOZT=IfG-<93bGfLdKJPuBA;(vzcA$UdTC=l z&4A%J6*qsm;OE2?jvWH+y9HC-!a_bBCb2&tHQ5c6nv87cU!Lu3)$k1%&o7?m1T-Jf z?ux*qwVs`cYB)awEo&S7zpi%X8PQN47O?1aZ$dRqJaRf(BKRmV4H=cqSU|;NcyOY% zQS`BPpGL{3PzH=+R4Ds;_lx7~qe3|2iX?LZTn@~r!3D73{y0KZebE?N64c7)es_?!lY8a zMDk0cAW5`~6hi0En{fE7Z%vp9`j0BK2mlK{91ru3x0{%gDpF$WmalfXisu80Z^|+Q zAE;!BpfSj$E$kSJh@+TnFAuGc3UjI$d3rm$___sEz|Sg0S-f4ml`)gIJZ0bWEEXN* zD>i_{>o}>EcpxIdqJCvD5sQh??VXr^%te`X#xSBY-KMTX_=99BH+94T!fJN$HA(4v zpaP`0o$dY|KuuaQRY0DR`vs~9q>fNklf0m5-W>;R;x0|IfBH|Hfaz=cUQ?1xT1op1 zM)0;X?p7=a49axV7MpJmEJKKu-{fp`E{(XAd1@ZSuy3IXXt`mN% z6MD-b$1d<&j|aWA2kce_u+`G+ee2M}V7J;={}9M6&%FAfwl?Ok1!v1-ly_lmwRGHV z1qj_}wi8vW54Fo8YULGq9Ws>%3oC-JNhk}jFnLCm^WEyN8&u{abS zR}U6fpGw|~!L>X7mMP@isU_@;z16E;gko;(h_~fnYIP272-a2uPph-V0XSPtEUnHK zdoZ^8lUY4#<-u^ZdVA4_r}ZdUT6^PY^`9$x@UtEXJL}N6S-sO2fS0u+R+iVAu_I2F zFZE?dj4UsL-!AxAI6>V@cNc7|e*CNb@>e{VSDkoQJ0_*z@cVQdWJ;l0jcv5;D2J<4 zlto?^4c!yW575B0`rHM))*)aDUDW`Wo^oFvG=)9{v35no1w-;h3n$PtfXJl z2gPu>b3I&rn{}D|Lae?E)t&9sMs9rGmI8n?7=YoxzP1*a0GL;x^jehUfnv@KaGEK( z<@s0SGE*fM)G9VU(`yohl+Y@1XkHu2%n=L%uMxh7Tw;!CkMH*hdtZF<k9eu0SW%dC*QTB112T9L><7WtuP(yrWZ!@TY{vglGIDic z8GO#9;;PUrBj)PPODHRJHKnv9etMSDe~OG;emyhYdiIfswkIB}rI1(e_PCDdKTFB9 zUJ|6}R8MazH~M#SK?{mH4sak}+I}SvwfZs#yefFbQBwBMCn8|`%hw_vpj0?7S(g15 z-NSSIous-o@NW;sU$BF5-O3QFrXMh} z$S*7sF(+oRA7T{W-dqhD#J4wBb`1JYvDoT9bkWZ!57YgDifXN=zTD7JYf)};PrbdT zTRLu_cIj>P^8NHf=#~4@{ElbdSgz(S)=z`uEvhRXz@c}xqMDJCv`h+~dy4)LyX5r( zgObH|_2`2qa4B-Zuuf6@m4&G3U4gJ%TN`_gz=wvrRyc@yk@vUw6wQ=b^_p=KVK%g4 z#j~l^4iPs6$gnpz8m#hE8qbyulswhEd~-n@y@SS8U<`ib%a#m^Bju+8qM}TfCw67j1lJEud0C)$%t%{8M zz<;+!b7bGpvwLIbg1mNqr6dwbf`v-Ft5+kWvO-=9q2!}UwOgq5|IJvnz zyLf#$P2*p$#KWWzlTxt>FMN_Q0Glu{v@rzHFZa0jNQ=3tSZNZzwLy~@t7k%^QtQ|h zqVS(yBsGDP)R^DOBtcpilCfjg(sc6SwTRkq(im!wPSJZ@gBA}h2FQn5ltZMAKgNBN z*7ZA0YR?+K_njVd6D)tn)&!tI7eeZ_DVuAF6%i2Oq*P2!@zb9Qar*=vboPdIls@Vv z?G0PBQP%ui`Gfro2Q{!}TCG(FT1kxvuOFYbp=oa+DPf;-0R5wc&<~U}y;)4Vq#05h*FX;<9;dSEN2?0Jo{w0LBEb^RhFIgN1nyqRNpXZ!m~{U z()?zeFqMzlS3TQm4#rircussAYzYZl(U|= zL8iIB&SGOBF5kW9jeG#@?~^?D2g%Jn$Qh>N?XHK^d>N8x6vmwdsA9;i+zi5;N}95Q z$r^gll%UjqzRg)SD`<4*^@@!8!Dp`@?BVdW%LWlSOs-iSD25y8WA4U}?6>W$eABcf zRfR#CxBU0`sJ5j5+MmCF^hP*`JlI9Y~lCM zpHIpEoV|Vp;g5Nt5IY^esvl@H>_NEH*Z-NleN&^zHQ$6&V@1wJ;IO}&fB52yDY@Vh zunf2Xda@CjHe@&x1)+eb32Ku6Y`a2eg#r*aF-?FkSB1ga(W^J%QGqFl$T~$29u=`s zf7pjg4TJ$7lZ4+fvQXbhQRWrWsR@wvwrP?fu@m)LRIK=baY()P$;MuSr#Cc011xjb zsEqBy? zgNADBt_93kH7SjtJjIIAOcR7*>OM225JuYDW=iFqJ|h~AJcXWsETILzNAFTf z*KRgfZ(on5(<5XdLK0O_Y4*JS-KPTt80k$MqUPHt)<{gL7 zwaMhnhIbVz%j+OjFrdXD61hLj8LtD&WxRHj&Ul5&XuQHCHC}xlKD-)x%)0q-_=cq- zXIxZ73!q3LND6AdWg=7Z+Aw9@Ty(BPQWn^F$YOZisUv8nLn;Y1Xl7rqNe{bM*#uQD zyx3icr7vJ14L8?BiicSe6emy5tAslnKy;NAOfG1|p1BqVmakaKqz50~21DAMw-wD| zgUTE+35C`{m%;;;;c?5FGiS@y-NsSk z5c`g(m4#)h*Bc~g0xMjLQW~F^Ew~g3sn6V9iLmtrRX>*c2qGQKZbZV{J4|=C(CFIb z(jATL9g``$V&SY%w*eP-4cL`aWvq$dD(?ey!poU9JYs=%R8@Uwq+Nxk1`0y+Jn=f; zgJ_+Jm|d6igsatTu+hKmWemQmj_iFn)THt@wAlU?%M=H_WTWDao`yd9Y;G6re62x}tvL+bnnAArVaywOJ3tvVFgfuA77-?A)$2t{0)D2!w8+Rxw2c za5mhDqDU$*`7|RUvke>*nk4o+b-e(8ZzVGuz_;v1XUk{rY}q}E+MW!N2iVdleWkDJ z%L*bZuyzl}8$SVSe%q62>1#KlC@LjTNomu^6U1oU%PwKf(H*#njF*>z|BSW_jd zpLdl~`!mtM)43=Mo5SF7cx`*fKBgC}&A?_V&u@CNyi20h%jGlQ5!EBbc5teprOeyv z=u{h>YM+Nw?FyZ0T9PJQK~bFszp-Vg{8qZ5ev6Bm)1i91nfB#q;(eNoFq^iX#f{3~ zxiDbm2l9YGTRjJumN6gIDI|!WkIq5I1DT5+vmRcc?9eO|C>hl;g7T|lJjLtv3Yuk3 zBj&BfRbQ204KOc<6N*4}C##-@JiS?gsjeythmvT;QVM^*)O72pu+?^PMYJ+kFf3-m zWU7c*95>?fp4>n0IlJ3yfLn^#1>-Az*Sxk%;Z>$`y;$m6@d2su^;J7yVOt|*A7Qv* z25JK8)gpKngRw2h9+*00M>k@R{^9BvLEF1_A0Ahh*rxlip<0CO!hYprQ^5#nHAn?% zsu=;zhPV$M=U|BQD8PN_Xcj@v*%0@kqhAEEi69N6p=AU#`&qjW9ie$`Jc-+`!~i0= z_HCdu7KbqE{E2o9Zjc482C{i7|LQX`6FRjB;rsnweF)-1*lcE zDav)8rnbL>+>+S8gQ7CoIz=ycREla@bt$i{PCM-e}49}{l{;g zJ$w4}hrho5=JxvXZ~WQMKV)TkhyVKd=@0DkZ-I?Gd-mscAcxJXq4DvomH-?E1lnc4)IzFaVwINu2upp4irrhhwb_G5*RVt5S3X>HN^e8pGP~e`DU9~%X5vD@ z(p*R?*CaNkvGsuZ7fu0;y^H=q|6(rp$zn`d%*%8@Aq{xmrN{#msWruKZA=9|=6B+n zqUJQZ!&6d4P6SqQ@@$_xF@Err(LP%snxY{PLc~nd+ct!KYM6h~g&MF|pPELVX+7~Z zs0oigeI&RD0RJa@rF9&yGpbKYhReoaiMWyQXS-=5RSE&;OjAx~K@t(sB%oX#X=*XB zcrNW;wT?i&g!C6t`vbFRqWlJLn&%0Rpddb+RAHK{WpfKPec$u6Oi8KqA{Uu?#@njd zF;xd2Y%PB%j>0o&uz7y((+Or3C2La%y=m(+Xx+=Yl{-*PU@{~MP zjFC&br18$ouP&oCQZ{47L_byW)XbNNJLuTy*$6S|Iar$&*|q`@G7>gyL^Dt( z!6>e$R&}F0l9n;*h2RtluYN|6sA!5PdL7V;Vfy=hy?$<4nkTdfkL7ssy<;_Zysvss z(=49QL@&GR@6P)o1*)Ce2*tX!Y&EM;kF=sTXpa(H04$xeIIgj0(3J9!F==^pT zgiqPj{>$5>ZN`d$LM$=HzDoIk1rI}T?9JXbHJxm`@BTv%T)+9J$$vO#2Y-5}C63e& z1VDtCH{7`H3cT&zwTNdqi>3yj=N%r53CjxEQ(@~`#142l_F1yO+vo|HvdV57(Sl{Y z0n0WUHWP)xv0i|CO4H;iSPQZE!7(X>0KNmf(A(x&M%a6)7pgEz=r@@tep4-d8$<{ihamJ`aEoiDSwIE``yLXQS)Q*+%2h${*p$q)!r_K4 z;7n!&YV@<1lv%<3(MS5t>ia+jS4p{A5LHH6DCUw)tUA=U_}dt&DH@au6UDB-w4xZ< zmB+np|!yHbutz)rKnsV1Ev`-Y)mvp z)cX6XXCS2+qK8QqiA)u;j9{iHnM-lUvMMz}OM_dV5=d-R0M3k!V1>{PL82ASmaI3h*F)iqz z$n4j&Le4S~FF-IS5+*EXvpNZgoVyUsz%N#oMo>$sn36Xl(9?WW2|UD7~mu-VdRpQzI}rq;-W)5xYr^M z^XF|qx79D#$rqU1qrkj{4+Oa#QfQ^v#JD9ZdcV7+xz=7M719iJX#$^{JP&b+1Vj|t zYoMnsBwy8Dnn%ro-9j6cf2wy4t2c2QRb)^AJe9o|*DUBO@|%W&W0c|QM2`tMPwf$Z zY9_?ij%q$0;;<$g9|X;;{gi4Fhr*nRR`b`IxJ2N2lQO9nv=m9gB#_IT{==3*qlyV2 zuAuAdxHA7#N0p2{1hr}PGpewfq>W1evzLuZwBHPK9M=6pfMjUnEL$TnFPue#2A`l- zmwH(G$dl1|q6NCM_^SRk;P704RYu+i+D zdXXlbt&WFv?NnhQ48>Nj+)6mDBMFG;7#h<7k+cvG7WVW8H#)n%f|u>Na0&mg(mvQp zY_B=f?4Yig1cY|KF*rmIujU^9E8un*$6E!_E}8yPaFa07DZm8MY;A({iWQgfB&V`i zqmAac>V0s-ZrWtvOM_!V`k;i|t1;g|I_ohrZq_O`0cJ0Fob@a})+j@AH0{+Gr3<}b zEiRwvD8pkmpn3=QR5x}X>`kF}I~y3>v0nf+Y4XZ_^d`+}RWlGObOfXUoG#oXxJuUr z`-A@4G2z`7TKA$P6WPQh>sQVjqj%c~VzepHEKIXAN*nanZT{@~$}s7tnaGgs;j>|i z8(Pv@4>#CMrG40Z@=yuulx4E&5zk^R4?rw}Aw!i>!~(aHF&nx_$h8PC+=IkhwGrUfu3>J?x& zZ{T!6VW?;1TLS@K$(JfGyTd*ZrUao40qq0mxiKb!DS3L9<(w6msh(HZp48<@Ni zg$Yyu&6<*7Hc+y4N~6F#ALx3TvBK1&vOa+u``vFO?I}X;UglWlY&^>qY{je{dR+(< zsdc;tynm!jgn+&e8OMW|SVh?>nK6r*;|{tR1Qy)3=GZYSC{I)^OK;odcW|QvWZ@99 zI`Z%rZ{4aA;f7nL>ikkC_}LA}uO0|?ShcxbuNLHnMdF@GTmYBvbDAkGvX_)6p&lW- zRBERXK-D$l69#%AD~4{SJ2eC_wZC+mCNMOY?4FA!RBjz7-LvCw7~Q9mC3<|P(j!z&Bsq|6(hFnCKesxF?SG^0y4shXYC z$w6N9W%nBBtRPh?{bb z5T6nU*4J|{!xO-Pf^p64eTB8@901^?{lGn07@L1yra%Od znuNWN7>iBkKRi?n@O*Sxr7yBd&38|*wQ~U;X*cM z@L*l7eBeE)KHC?z1NKvhAgR(q0L+gTY#9(})x1(Bmrxxp3WC3?uYocOuN~L*u<(0( z4KXj`)enHxeA+IWa1D9rA$9o3K@oZdiB2EpWV^$hZfkDXVbnTOnrp7}f31XXU| zlRLJ4a=gixf?IrYl|8BJe>QkfrNco?!xOj{SiYE;Os!27c$jMYnCGT_Lr{&K_B^5J zP8n|N-)x};C@Y9gGpC)}S!Ob4w~Ww1P3~!^hLRg$Nmzpz!X~819>ni#%E{ObPGEs> zw-qzlQX7priOhl9(#+X52(yd7RVNz-wJD@YnBEFc2K=uYrKKN_ufFOq4arn4FhsYN z2m>^j6+l|su)TuhrObsgxKEpVCX93Ps-m$?55=tIYJ)%usYzQ7YVoXDqs=&&pzxIO zEsJ(S1hsBMN%7cLVclR?F|qvU2;**yO+9p_>Z4TQ#())~IS@#P??<+_mtqLQ?ck+X zvSe5>8TW#ETys%oX5HH=U;9&3kKWc4sJdr%L!B5H*2#sYlc^YBPX>1m;xtp)s zIaknUV!?2x;Y-IjckMBpL$c#cgf1o547aT+Xu*yV4{?!L*py%)l!}-yOa$S~afTja zF$q%-!Y#ra)T|}%;hd=W|C-@N!85TL&Db7Cx%(RRL{dJIB<(|3vR4bP7gV>m1<)F$ zHUkRC*o$}SE8hV>6$|n*3`Wvvk*CNsF=;>)glADw#uz_@EL8KV5I(;HNq5pl>Q}7} zX!BjF(4tTV-^O?$v7%bDQCOXYL8A}Uyh9Gz6PO*(0|DRaePq#D#G(nv8a2(2FZo5RU$L1%4zrHPvoLgRS7X$ z4}_*YGT5|cdDh^rxb17m+Sh$$UI0#Q^ZgnCs6bc0b_~ zWFUbKxezk4A)Xd06oKRP%ztNOW0@7oE40HvJ(2w1fBz;NZ0)Fir)xGJgiPWG9`_e{ zQ{<^`=$UUM0jAwSNU(n6<--Gy2M_}uZxX>CeKiq?rUKzH%L=`@rC(2b3avdOmmEv- zLE8Y&lFoTn*LSQZpwA1W_mwsDD^_69Fev&UiWN~jEt7&~Oq42FdzaIPoP>Viny~k{ z5yFIQ^>|a*LfRIO?1c3m8*G3|opBqVRFbzUn#bO=$P`=2lXA&BOJjXdJp3BkyHrr* z+dq{K68^p-VmFBT&r=P!ZH1>NG0R!jQ_o9ezg}wt9;(jb*yLdK)&C43CEQG8GW8sGU z{MUxp!nZYzC5w8h9|x^|)1z;{y}B4mCjE|81mW){sRBHH;O61~DBU9%l=!cRW7^tp zl)bxX8L`@AdcoCQ7!23eYt60o&G{uUzXqOM+UH4B}R`nc9zB#{~l5fpX#)xI)YI^*7O~Vr(lCVKrcMl6e$a&r`12)e3+dhYnK`_5P zq5oupD{5z*iWfI5?+iEG(4_F;4f`)I^bbODfHI{8p~y_4GNC@d5n6wzpZE1pIfxkgaes_UJGhM?1wg+L8IBvcr*wlEPopX%RHB=s7K zGf~-Th>!@DXQ1Y%)1xWmWE@S8tRj}!U`cPIMmE)*aq*j)3vWuuTbtbK*Vy=0yk-?< zSwmR1e4&rGfZeK`yPY**n@fIgYB}z0i>|v9JjKrDw8y}kSNrz^cjJIw7>F;^he)I; zqS36Xl4oHBE-TKl-x&HiYwy(eb5}$GT6AvN4gP8$-#DRGKbw?vf9L`nYk*%3LXI{N zTgujhG9vAy>eR{2<=MsS%V`?#!=hLg=YnVc5)B(T+V&N@cI*!hl9#E!TLMNKS`yXqH)sPcSv9biIM^m4|>wYFDUp|c{VAy7${&r_n-X~MI+8pLQJOp!!J^bNayC8PfQf=%hL z*9zF1`xk?(%wSN~`|K~4O_*R0_b~tD35>2sWB;QI!E?Ea6c1y#@MweEX5?yi zKD%oAMRtsz4hW$39+12IB3;51MspZlHCoa9tSnX+TtN}Wa0O-SO(zEfk%Emkf^bb} z??SEHIRDGpHKC=ZUSVJw_tXsPG0rf@dGH9$dwYz}w(uCEJ8bxT!;jTtd8fq=ek^~R zre}lwSbqQ22MERAQ_{eW{#Wr`@r)Oo7D5j48#LakOem!)M2b5M%$U5lZp9lg=3One z8?5s};S2;czsmHfq_1#uGyeIQWbDE8iQD(de;BhOpnW8N`YIT{yoM^Yh_A+06p1Wj zQ6Fe-*p&PuobB;{WE+-J9O{3SG(qycIfR+7N&fKOHD0qFt*7BlWiKgJPqY`!%*YB! zGo|OP=`T4lPR=9pw1{%AfA8QqFiDxY>I(Mr7N_G?l>_7J(t(|Tu&To?TITs`e%$_F z!#(nfD+tZ$!=s0XxiYY@pt)V^LnqIFqH!$0LK(a;nw0EK7`IRiG^@PWPm6VKu#Rre zub~x1{`iMe(Byvi;t!|Z$^WomRCwzv^P47Dk=mNmohTFpz%CWpn> z!w}+HYJ}@)V{Gha9vf@Z+mJB@FFf}pNcY}ma+v9XfBU!AApYj?FmJ1e;co{y=<|Pc zb#gO1;oMLalhy=&VN4Rbcr*Ly)!EmVuYhhC-QuxG7AvEb?-KVI zzH7VoSMRiCp-#JO!(Eq=H!o)mPu-SBuG@mL>8acI=#(uwDey@4PN)8KOShq+RNZOG2_^k&F5{Uh{8Br>#L@OkgcLr^FPnFG8t^Ppix_GBpXLVQwL3} zdwn0S-q2>5DJZ|P7xRP`hDC=~uWTkCcaW>^K(b9rLY0-eCVb7!Qhp51yf>}Mqw(#E zZGYdJCF&(g>$&eRXTE-(QLUMsIm1d6BHJOertQV1)BYNsIPWjmR<8(*U#?t9fHQ1l zqR6aMsp(SMV~_rl(&`Us(%*Nh=J9|;HvO=r`PGHdFl{E9GK-7^tTM8_(jHx+$5$6S zZ%iSJ2O)

    1e*P5kX__Z6=X+pGH2dy*Gosw}H3UwFmtgadl`NxCQ-wkjn42u?0V^ zt+%_B?5WC7iB0clcB;y-_xS^t`~aR~UoeIYJ#jV1p5!;)cIf)d1hf<+o zOvFBo#(M;*N)kOQB6DG|OksuYaC?MkN@fr;Us=p%qC@=- z1lNI2SN~(q3sd{qRET!Y+J09_Ry!p%8__Joe8sVgWkf^;O)Rt}D;^m0Ym=v^r)YaT z{qoB%2>^t}EaK^ABLtuN{qyHj@;_&PRWH6e&^cyXtC-=Sb}oNi-l9#mMhB{Y{3z}Tv-5ey~YyWQ%N4oDQFkA zHE%G?624PO5*WfUN%$Qj3kA^Q{#+7ULs>GVRen>w8b4I8F2%e>hz{$F48XK)dWg(s zwqQmEUCRcNBQNOL3;|BQ#y794R&g3k0hyZd&_rZQn6)1Nh;W0ASX^cCuJay`O*m#$ zgwffzvQ)*_Wfms^g<~6|eR96l0><{5)C>z}9aMi<&(XUnRK_jp_TJTyw%Z>IXw{g` z^?srmp=BW^P=?+HTTjW48G*XW$58h;q>#DMptk$t%)VPoIi%L@k6BB5RqyroGmfIU zMb!%JYDQCbthuc%aNMjGsQ5Bt674Zzpiy)Kvkgf2SJa`kjM2MZ5xQIgy2yRVEM=Ne zB>bOs4Z{^9*O#v$1{_pkt%?;a!{o-Wn&5f?)@tySkV_>BsOYC#HWQ?(!r%~0GMOhL z@|P39D-x-}6v<)&FOe0au|=QOYEhoEcuL5*-i!+#(IN=UP`@3sxYT;J$q`*KB4o_4 zJpO`2PJ>1U1#T~p1YGxU&I?K98kHA2cbZutK0zRbp;w632-#PDb)O6dw1agv8-Opl z!JqI|Q1P}mml^_U2Y@*!us0V!LH`I+A+j9{0w51W-&))h;PSFzmJf11UQH<}lJn z*;DmoeJa*C$9z3KjPdN&6iUG=Cks<(-vCm!bUd(io9+1MQ&}POpc(seOc;$; zS_PYck`uV5w8CjM%L_f&%2&22cLq?6IE&Q@m@s~qD#l8gAfwP0LC4IT2w-c2GkeYc z4C4>g(WmzA=QJfU5;5{EC_4%GBERoFw%X zk3u;XM_^vlqFo8qWFLnXb*L}JiZaLULWu!ATBZ~NGLw|bI~G@(Rp=~TYvNVr32B8M zMB&kOqrhgkA@^uHJwmzJQj{~OjX}1B#uk1}bA=WWtqXxARzPvsVioR?!;1#sXt~!e z_UD1N=PtXm8K>DzLo7Kv9n{l6Y~a>zA3=;t19Yg0S{Y;z>WH7h1UQ2TFwCgs51^`o zqz1Ja6i_v&#VYu!C%|e4OL9jrmJ@I_s8J_4>uLeaq_qg{pf=r@x=L|V=!|!ww1)Y` z>WHqMsI5WG`XanGqPPaN>xuXZ0Ry*JXb9NWu!2IuHm~q=7+%fjP^a#wY@Eynd4!!Q zfzxR2FwZG@Z3N!H#Y43YtVB{4RSDFcB`01mFl#`>O4hc?TAB_Xwh@WpLfE`ITtE!9 zxDT62YF*#a4A@&lB5-scxYPt*Lkms>HXBlw0yE3x6MLxDBp9yS^G|O>o}quqQZAe6|*aE3_y$0Zing z9@ck5)SuuJ{qBdSt**V)qOy+T2o@Xegcic2EY!lc1_upy3Yq+2U^I_-AcOXMiXClU zP40eUrJu2?qBSPG*LVqAUMBtuoN!qHVYolJ)k<6rEGF3z@h6jJR8Ea4Op9e-;*4NGf5(_(& zy-0iQ6p%)nB^OS}ZZv zx-w{%e3qDFtM&{%T=^_9*19sR%>$&iZM7Mc>Ncp!K1&SD)u0ypEitZjWl+-|OH6RS z+;*Em`E!FUQhrMeiGtk9_4UiL#5n4Wm6{E!%y2`8Tj9-`X6o9z0d*}lFXPOqj50zt z0Je^frqiP#vr}Ogm}6U6EGB;>EQ>^JbAT3H4(y~>(rX)lYtaHWS=oAq?-(54kOA%_ zT5U%5%e@umT1~F1{s&aq+Po@A5~DBI8RH30xyc68?3Ag2euk&mz%2?Vzneu*T+2jJ^5xP9fIKs34;vpxs@RMP3$0 zRt=s?!|E*lMzuSStMk&}CiVI1d4FDG^U7&aFqwTt{`cvRzx{dg?9Wf1{%|_^_rLx2 z>5o(R<2TR#{Oo7@kKaCf_Vnowe|`PU?e*o~__LpX$jbB%|Mm0JAK2yJ0vmbu?9cz- zz>YS~1w1PzLQZf>zk&$JLCslm@3*gO(O<)}3_BnYI3#Fx|A98{ZpXh~RQ|61eU#JR zaCgh6;zAkFIF@}GXk*5gI?T)FW=xBal*dobvivb()v$znV5?)G+GiZ7hBMtT4M6>EhAm;M1%Q~^v@L|#62SVf z-Q44bu+cK+`WU~%8ndsu;Eq|$;_61=5;NSS(4(D_Ga^~Ub8gBcQ2ZTNohf`A%5Qn0 zSh7H^h-P)!Wa%7=&Eas5E~7dG9OZ#4M$}4_NlYaByYWYdGJG8Wt<#%E)DV!k8p5!S z(wrOYnaAK$?0wE9EcO|TL>BL2xY}x?R++#Z7HndPOIf9=-_wN0yjWKeVYt2Y*`v2I zdwto3sp>f=G|#1=(TXXj=f?wP+mx`cirXgTl4p1*fR?qEn5>Bu1ucSC(Do4BRb@Y^ zKN8p?s%(W^13Zf`-Eru^osAG~S`tZYp*{b2s?WcX+3i}!0N7X|ND>}5przBo?N#Fw zFTNlvQA#zv80J$d1bGf`qDje~uM3hestWS_l%zZ>i_j7U&XMI0UkuoW;fIwSiyF8G z6!^P3r36=QC2(NY>n#@?r_w#G7|}I+sbryL?mm+=`@jeLRr%hA{!xngx(I{nFwZti z^Zl3kaSJ}Mv)>M^pn(sp(E~eP3ei=TQk34J1dXx4)AodHJql3KTK{MPk-)s|ohfLi zcZwbj;Gzwi!p=wt=)c$j!Oz>`fnb9rTXI#%Ar!$Env2tvN)nQ!Wm51wsi@-}#Eost zNJ-Z_LLJjPMPwO-bj>OEGvB9GA6QlZ&HzNvvz$s=kv|e?O4T!T!VntuNFjrbrsblf zNlgVO_GRD2S+I;|Uf*dGMHe0?;vqw*Hb$2Go@E=Wk_4EHzNXO~%VP4avc!qk8BGoF zZ7#(<*NcoVKY>8MuiOQ})~0F2ZR_wgTvrhm!rBNn2I>NfWs#Jv}~ zX=@mEI-Ol%3iukoFIEOkG+XhX$1F0*#zHZL?jIe{*lwN92fXaGSI)|L7?RDAoX1Tb zc0*&K$k$M+?1F*P?#kLux@!2CO6u2ajAD z;`%kp4}gZZFSM+j`_I{ELS2am5)1MGAqV&R>;_(aWvwaBWG!j-wr7G~idwA+5Y5)YF?OiINj+#CBAG~he7hCsmJA)6N( zXTL&#kr#f~%Eryt7vHKryOE<-pRgi2iCDfmF|F^;j1EkmRAjsmG8~TR$i9J6FcL{( zuwul5oU?pYH8~jd`oQB42ssHnAcr`Sav{7fi4DkRA(G~{f?se~ISWV5dn)REBFpH& zknEd)gMMb8wFCS*`0F2xO#L|Pkz_?r3dknZx_Jf8=Z>S1mc@!?1&`3OtNH^DUDide z7I>G){R>e)K27BKwbIpjUO?wL*2*6Pb!=!mi#1oyyF!&4by@{OI5_ z_Z&mIFv9;@Egq7IxZ6=M;)6F&M9!*5*y4SK_L3ItfriU$^SmV=jE=^!WJ*~MYF3GN zh-cSV^zK zi6yr{V0!r=VJY|4qaF32&t4BwD!I()Jn?&4n+v*ng}hAYk}&wkNz9f0Kbg&5fhVQR z%vr!o1@J8x{##{~B2(CZuOg!x1pMv1LwFWjKv1WCoOz-*Se;hsHyCU*XXJLpqq}Rz zc_O6#EpvVfHsTC;FKC^aVOq!+p3)_ zil?Azy89!^K~mX`0;#)??rk!PW9R)!V0S;^^G@_1e!K~>ZRVB@$2$II+)Kb)a*>yb zF9^*c^WOdk{Qf>}C@04z{0#0U=Co&b1y#Ok$>y@F8ctC9v}8UtbrQC`ZH5%O>Ppb^ zHoQ^T>JY`Mf{Ig_q^i)slhZ$Z@x_!}a0viIyh6~fT^3X_P)0G!3C^VbY{4k;o~l$W z*;M_0$c-vg`({t=m@O5W9TgT}*y~+r$w0sIW}w^L@*a8y<(&`A#qP{D@6RLmx9jGZ zA2fS>K+d?E4eo6-BvX7l2fTN>y7I$&GQa)&ubCBo7*vv7u}A_OZ38p912eV5lRAR3 zvJaJuJs|8hEblrv+j>AHN^g#JN8WTmvj-K%cs+th#%qXR#%$*hy7twIZ00xzv^Yc- zhr-VtSR&)KR~GZI+~pxj0AUFL!P&_}ifMXtlDG4W16pj!MDEQl4rsPA1Psb44rX*= zxdr`M!vUEG2TLXh%QPLBUs`95`n8+k@CQST>4OXzNA)|c<{4Wzs#m=*@tAc<^+ISK z_{gVY61GnJZ3y^P(Wz>*+3DWSHk_!{&sNbkKGJR{>gYQ?gzxl_(WD>Gix>Ygy?Clo zThkijfO&wLCgvds8Nhw3{dabKMIy-{bC@OyL7_M5*%)$rTQ%CE)#+$aur%+%!5y7= zpS=_BAP#|5nbK^ccQCl|9I5K1z@`wH60=uPw1VXiQrgOl|D$9fMbc<3l@I7Ef%bNw z4{8S+yFLW6FG>Ej%zXkSD2X5B)wS`{AD0K#4McGDH)dfxT; zOgj0Tk(5g*r19ps!|=8Vkth0`$Rr+(q=`?MvBE47Sbl{dG{Z!rK3jne429uaU-6u+ z=sg#u#Fh&|o|s!tw5(hUe|c|fnXWPaftspi&Z^ZAU}TPkHOx%_=YOlYYPxK>{72Rp z?^Uj4?RVZ3v*s3X-)L%%_cOS)tKMj;9AE~gO(yvmx}#rj?6c7RqV>kg_n+`+eRVNB zW&q!_(nnSC0{VCIHB~)q@HNfo5{(Q`XV+i9ex_FnSdXqQHa*a;wuK*fvwvFS1_=g9 z->o+vfeZ)8+_dtm51Oh)a$av1tia#^TN_7N77`*o@6-5hzk&1z1=PLXjluf9FoC&f zx#4ZI+a~ltlXH=3cH9of)|ieb=QajKVICj8W%Y76tWD^O(U>G+NuGXx_Uc(nS5Z99 zGMeY>DfuxY-=Do|w8xbR1dAOHIw%3xQlzX{v9e8U{^>u-i|LEg>8Wn>##+DYSIGT~ z>GS7Pw7S!|xM$OCHs?F^J+OHXg)YFFr5a(fRe2&%wCdhGusP|f(&U#^`mBRf(Y^>< zyY!da+iQ|jNmEuZiEaR27y7#aU12n4z9hRwUtSsXOY;_PWMf>Xaf~7eIx3mUMW%ek zcES#u#+f@~Q7L(`MrJs`r3ZQ-LcQvCCbpr8VXHWle$Uj#sK~dOVuco@^Mq!1elZ?+ zF4PZ1#jT{7(wc+q=?;UPK}(aWscO__xP9l?tx4P*F1bz^Rg9$cPHQ?nyxU3oE)_#> z9#+}V8aLF=;L6Sf`jfx{)YIr(iU$SDP>@8>*xj7gMTDn&nrmf=_6LPvx;|>AAcM6$ z#<)XhQk8T3mx2QbkZIQEjz{>hVsPK8{aux#?}3=JLMv-yc`@#HVK zzfL;m@es@Hz1b340Y;DQ5LdZfYX3d4J4?4SF%2DRTahzxsj|2(7m^8;@x|ith~7RT z@8?)ndIL(e+<)R%$tUEEnCXEXWNKe(F|p62LIc~g8O+@A^3Nt3gkqLq*pA+7Lh@x_ z*4%z`+Zfp6PwEBB`W%NdJykI7gG~K@&t`S%&rWmy-?Q1o$sVdFpC-xfbHC4|HG8x_ znF1`Zy1Op|)gnxRfW2|@!XEsmfhRDjB80P^wn)oe<>U$IR8g&Ly` z^Qyzik4^KQX03h6+g%T-}L>~ppJf=l$bSxxaE}8Th z7W!Q?JyVoYExs~b5FIS8Q?pO$d!Cjl$;x!jq#n59X_*u>W1`eYqpIPK0i2P4GARt* zQbujzen$a`1yk8kVPb;7tjV)Tq}|K1{%kwl`r}Yip6PLYMNT~=?G8=J>5|>@BI&SQ zG{||2sTuBBNWV@B{eFA@dv<1#sKI9*EGI-qw+h2s?Ms zK*zAk`k%Hp^HFN$htgI&0wW^X=A1PfZ^;?3qQOL{-NT?f=koxT1P(w zf^S1Fgs28T3;A@IL^JrSVi{1_Y)V3Ls38&v%8}ca?wGeG0A3^8`PXJUTj&}Aty%10DY?&D^Tl)pHE*vL_JIMVtq`$ z5ta9%**KN`QSxW^lqYa7+POmla<|)jDH573;ryK6El(o3&I@r; zB0EWSFrl#a2B*22}u%JAFvK{qvqTlNopg*5CM&7;d1J-*~%6oQQ z7kmpP<7JZQzawnGco^KkZUR9Go;8XZ)5)!o`6-Z<5neXTki+Nz_;A&xSdGw8+#njeo6j7ENdtPq61rY16G{P^E5XRREVo7&or&p1|hgDl>p5XHb;s#qWxo*Ct|Ia z@%Cc%$HMpmU36D?#|dFen~F-Q6%JSwtYuC3y4b0|ZZa+Gj+<16*+0%FCBu4{s;_)% zmElx(@|c_@Ocp652T3e5OEaNv%7n?PBeC4?8ho@yiwuwd96VcEK=K87*Y=fn-3Lo$S4^@d&tDvqd5NEWH^mh+t$D-`f17U8#@qD2 zA8!nSDjH569_t`xyUs0l)6X)IO$sJcoPy@g>HZvucG+NnT5v!xE&y=V#O{uI+efn*)!v|=heF|Q{ z8KG^IgHH;bf^YK#v3*`Zy7h~~&dQA;qHvGFw=TmE(QEibcP)lSr>|@(zHBV0on>a3 z9cJY??F8>>tM!h@z!zp6zyWJY^+jsr018?8seY<;N6!MK9TQ__ER`ze=*6q99?FkU(SDkO+ z&iD$<@%RH)-f@2^=ami4HVj~t^;bYSo*9J{s^k!BPq1og^{noX9g%%h&mC^b`Z0KY z^Fh<5`=~h|lmGkwe)VtvVpp`HvQQ_HjNw0%I_XfB2~*QGO_P1Uae8`s`sMF`kN^Jz z{{QsF7x@3v=U;sB#p%C2|Kbl{{{Hm$&wr<%dvW^v)8G9YIX%n}{a2|9D*x^Dlt~`t zp666S7YomQ9L#C;|Ig(5b>&Nu^kItrlu|Ki>cW6)rCt*gtgj|y>u>j`(=T64Pc6u9 z+cT3$#wP~SmS@XpBqS3`f90pPmG`U3Wb&)p?f(^_IcM(+mO;uv1>*bsS9d&%zarrOYGMMC__#AA@np|uj=SDHoX9xbIP^RGzzL)fJ4IW5={P=N5?6VgI*xP$bm zh2BkR%GhhQ=&vm|UvYD8%#%{mWNQq_wpT(zeelNjrEOdBY*{8$Zod30f>k}gs>?iT zd1f<7bXyZcx%=nXQnX@emE0h5mYrQ+efPUr8L^Z`9v z5*QSon%i|CYZtD5YCqjOi?tz3o32sad!tmuPOi0~fIeWr$ndzxRc-cfv+#=rpml)2 zof7o#D+^Z!-b@Las#s?HpHc%%VXks^gQSr+GlXN40kv3VXT=P54~C|gWH-R~(m2hIi2iKw(q$yG+qY08pw z+EGsbkQ1OOttOhVVN>8T-OY!$!wC%Y>zFSLwz4TkVo~YljSY;O30+GY_3)Z{d$(l{ zm~Y=1#1t^n{c!D@x(BWJAYs00&87f>1=nR_6Y%sJfMVsznlQag;h)5*X8?YGa?&;DHf{owo^JAbd7zf)q{l9)b-(%XRlbc)JMDLyUu5>iA0m1 zY;C5WR3WwI)fqh86gAiQ`i*&eW?r`CiqN?-U%PQ5-FqX8D~A_fW>6}8zIJEG>M5N9 z*#=A%@VN;W<}Vv8#WNe;o8}_Z6758jSxW#zh>0acCopNfIseR`J*9IYi=Q8;%%1&u z=S>9+Bydw9Ll;>GMGB>AE0|K1<2S~6e$b@jspe(?1~ zKhPcF9Tw&f10N9C?Q$4+*MY?|Vf$c>^0r7U_$BzPMFuUt!;!y(CHW58eph8mDb|zW z8yL&Jb-?X>Fvchfnp_8GCH7I&kcL@sJ#o$c_6jVcCpc2rwrvq?A>(y*FJKUS) z#op&F9&z`g+4`*)#Ks0&gu&BIb|j_d6#wyE4_r9Ks73=~%+5!tdQ=l0|2 zhj<0f?j2+KPe_@ch7tL%jO8!k#g{@>W(2KqYwA?e2-JlhD(eFw&q`*eh%O4Y5p8Co z2W{y=!uzsXR#cHFVboivkc4SyczrHYieWsT3hRA^K}Ck^3JHakp%nc=_$CUp9fd0d zB%8BKA&4l8yetqbb5Igp$_mLNgkk8Fw>`}sRvL$-V`*02c{^Yl{5hB(5-nk<^D;`ECzPyev#|2zG1^#6U3>(lZ7ZLhq4 z5$|7fU?0o(H-lba0ig}gpL=V|@7L&khu2RzmF#Qitg_9bH`FC#SY40 z1H}S6`4jfC38tX6i{(m>C}mlI36^w;xV6s^Prjy#M?XHX zs4LX*O&j;gkC`oy@|Av`QOV?w9k@W=RpTKa{bwoJ>;i>FU+~9S<6%d~v;iCN0N53H z1eh?9-eGNX+K?kdGRwLh(w^-Ls8i3ar)3v$_ z9FPWmG)1N<*9Qc#!1FT8CZ#%oVV}@Q?}16oC4{K2PfX_P1o2t*m?!Y$q|#O^hTO#- zeaKl898TxmjWK#niXQfhI%r`2+grn_ZJola3uC3VoUpE|;p_ZXrD>eUozv*BH~!*u z-f^J7?;1-E?o-u)9-)9;5WPZ**F8q*x}d^! zA!Y0OF{8Un)&)37hL@{j**%vcOIa4i8T5Wa`RTm5Gh@;UQ1EB;Edd;L5^M!xxgN z)zy$N)SKozHISdO975l&LR-6KO+g-3JZeNgLgXEYY8p3+#enYjIfklQr32!QhY7*T z#055M<9vk}Q{`29P0xh2Q@X9}UcsiESA7iBN}#}n^BSlnXUPLytBN5#W>jgxGCEv^ z!nZyR70T~gZca(p@pHJ7d9II)dW=2iw6E=Rpa#gaLMJ((5iO@mJ#@xDH1(8s_ThIG zCvCH*)rF5ewlmvQo8hcoofs`=pQ_e4TFx3h|12zLrDC#swQcNs`|ttwy=`jT_wF_B z`*GiYh<)#AE)tn4lTZ`#dGqNO8!|re0gcGbE;&8w>WSA|iq~Py$VlS-?8r$X9%cpY zg5cf?V^72bQTlmH2FX;J6pB1m&e$Fv9;VnkJvN&2lLe!NK60Kw_`51t7OkhNB28K< z`Pu7xJY2@5K28c6YaJE8is$%DvEtZ0jh2cjF9?{$Rzy#VSI775z8=SQ8{yjA*Ny3f zc;s4sb+Zq;r23F;(a-c8%)HniA*Rw^-*?NH!ad+#n_0s z&3^P7`Az4y9);iZHXYM~Ml35>XX1Q|k&9eZZ^4cLkQ=T28c2Yc@Sd#sQelDML_{>P zt)4AeRyb|%?8{~uohR%sx3||Xf=2bdU0A}rO5k0O)u?k|;JZJJ)o3A-L_FLujC>5M z+;eVYHLA^`M^fjsV4L{k@7}-f9EP-C?XjF4YOIy%C%Ax>Ngkt;VGXGI*u|~Yk6h|i zluNF9dgfV5nYfnXeOR5UC-2PD?=V#+5@|{&ise*NtQ96Pl86>0@^BR+$;s8VIwt2= z7dOOaTpp8Hr1}=n3tKWtXLCl_`zR@64{L*Ls9g7uXUi$XD`j!B$%tmv_aQI}%c^-w z#Qd0%tHSa@wNdKq{>Ai*-%U@aT_@TA`p4`w+e~PxuVG~%0_0S4p^2DPIaQjApJ0%F zPPv3+z6DQ!35pnNpLSbe$&%l*%z6R*>+!&`ctWdqVVtmvlEDz$7t>Q;s#bjk*^Dvb zdB{oIzT($>J!IDx^2FCP--N`5F}|>Ht4E4lT^vOHjqC>pX*sRkgmq^kZu)Td>}B#4 z`gGpV(U35Oerr5dsjt-dz7CNgiX;hC#<#ll$tmPy2c5bgQHn=Kl+Vk(avu@ zk{ibQj<2btzIxTW6q*d2_6>H{?9sc;v@0;#<@z2gnDj5`LJ*qgEQ@Wup+elTjC_0Z z%FD-s2o80q!h^n!-r)=H8by~3m&IFHzTIa$A;xco!?2kqzM=F~At&Lp+xKfUpRzvl2--{_-NH1P$I)W~wU{s+CZz7LPk#v#rn~4aO_$U47AL8u zK(nvvP=q1m-RlN+A1Vx*Iq6s865LhiZ{NJRJio1ch|>*^PiK?6c!Cb>`2arqQi}9) zU^#*(j!W43s-dxsApE}dHeb8-49Go|oX!)b$WyJ){d9fv_P_t<7<9Xt{Rhor^5!l6 z%aiOd#4Y=nGv?bzKrxSAwBbF)$F)x6>WZlN0_~yFw4IYj@<`UT1~kWhs-FG%tCN%I zZ}+GVFuVf8t*z{iFalUb!7M?HuRv$5R#Qy-K8%338A+&ILPhp6C%hmjU#<)q%TkD# z@R)(^muHb=RAJH0p0MaSi>Tci5iMvUmRQ~)TG4FD2u+fzM16Y{L1)v$q6JU)( zH7uwB);Uz>W!1Q!=r!22QvF4&baURDvz^BTyD&_#w#pesKA2oCz%bFu=Srvo;t+)- zSJx&u#m`M?R>?OvKH)LTOcfiuNyVN*d4_bt?y^X=QK1raGP3Kk#3 zZmL?;?_=48&bNCGz`BBE9eqQ@pqwR zFw~Lm9x!;bmIy6!z@R2yc@o-|#@aH*GiOcP)dPbA!^l2SCXi_XO9s3WHn0a+v;Kfd zyz|xG`BWEDqR@LBwu)ZO$az=JD+!HX=?p^J`rf&2Yhm(YFqN|~Foh2*mXUiV`C<*V zC4rgxJrs3gUQg^8p3-xT;VGm4*SW^l(vTyh9z?)8;jXr?}T=#+$;zQ+lv( zn`#mkHgI5~W+ic!5A)w_{@j_Ms5)w}I$RPVe)9of_7HL7>+ z^_lUepil5jv8a@6cE|JYz%Cc8>|JmL*}}-E8cHAR?_QlQjP`eIJuod<^^oKfhz0CHFU+W{KeA?lYwbm$FR z3@KT0cK(Xon)f>ERp@o!?dn=FXr;mw-7a1+q7RV~eP}SfM)aY>#{Ko6YIJ#AcSO^O z`H~kjc^fgBg_f_rXnUunh_(A%OOJa>O7C3Cx~+SCu!ya52dvU? zz2O@_i2~B@T;c@Q2-%7igyLgL^YntJo<6>hyWP|LK?5+qwH=ak`INobHf?)tZV_x8Ag&V1XjlP!IRhr^Xs>zR`|R5~CaP z7v_c>qa@sGjFRZ;>=-36vZpah0Y~@14uyqdOvoFJTgYSR}p)97r3>oem`W5u0O zy>t6M^3Puo{~Xmj_Zro^uFj6?-N>Fs^$xDjj5m!HcgBi4c#Xkq{%Z_o`(*{Qea;nk zcCCz&so;VbZ)V;)7;k3VMIu@{dZ6+bb!V{Zg)$y|@E=hqbYME=rYE-277 zDVqR3yZNqjTvy9XNQ~#y&k@NYA>%HY)gMQ(HlJFN}n^@S%{;bY>@1*Is=Pd>{lTxikMP=?LP1}^#=R2o;9fJ z`2CEWU0?amsACPs&Y^r#hwHJ1W5)z;@eldRqfdht)`gPPKm{!?VrNTerc z-_I`3Us#5Dc71h&n<--YA3tre@9C;YyYg=cL81U(i$zqXEW<)Qd-VYug%5Xanx0eD zrjRz>G@Q5@VX99kNNc}Et0|&89}F{`_8S)i?QEdE_Sk3xvfu{T^+Gb?LQ*O;2|TxP zU_Iw%FF!c^skkFK3?lTEHOQli)qOCoRIg)%zy=jcFDyo(ij~deh;6ONS~=^ccbW8n z$bR6nwcLnn@9a4(jQz?vU_qJef2k;V$(Es2Rd0=|h_CGYL6NYtn>S&a>i08pBN9f= zZr&Udnh~m*VMYsn&(M3Xm$sw6_LP(J;#`|V#h{K=X^opIwDP7hj#_l*P)03!)S?fp zMel0yf#RCyBF(iZY-NsHX?hu1&_uE07Fhuai=@mV(;$n9WGWX~E7(E>uV4`Fjlp3v zd#)v)?BA!S`l*&SyR-BPSu)3D(_%IBP5Xi;EYvpk(tWnYI0gX%AI>`FcE{zI zrsVsTZcHnWAjsiYqB-`lb!Zq8P4w7k;VHb4OQE+nN$DL^Boky=Qc1Id>G_1qg!);Y|4dY3%biHGA zWZx4unoMkKVmlK%lXPs`wryu(+Y{TiZCexD*6rW_-uvE9_pH_X>^f(6b+5B_S69`p z{XCJA?Oe3T9;T|wQ(#(J*3bYDG}5-PF}{+Ur;f6mH0m6gB_7@|Ks@Z$wd;<~^+A&? z`sfRgNj9X{pTx{2lOX#8>WgVkY=S@gBUC6lTvzSwxC}GV&(dWQ>cn2e&83)KueYH6 z$EbBS#seQLeYJI@c6#+BAC!a7Au%!3w>NN)ak{{@o9+7Uu*nfreD zl}9S#QiVsPkM4Z)rE7 zaMxpgipxca@$78w5Bqu(^3x8gug<3R$+B1bM5xk&8VTB6vZmDV7gCO z)`NIF_;p{PIBInx=JvdTJaDGLch}39Ky3Lm0?h(lmV+MIj*B7)<`y9=E@jGc&2OOk57Pxa}K*-We0)rHOhRHkV1?K+7 zC`Nix=?P#rSeH~f(kYseK9;nDpExZ=48fEULP52%eiBsYewFjCgDd2A*ta?liBwSf z-AybwEys$E`1kV35b&g{mn)vWT^IbKF6eD6=N?TyIQ1BQQ;b4vs(iiK9s6;QHOMda zINc@sSY6Rj;K@;6qRO1^P^q}={(_qGBKroD!bLF`vjP84ng;r{-;$auD8}?ML|g+#<55J zD~z$t5E@yLbSw^ggZ#dENr!UIh!uyE?>J-bILMTYGt<&$ z`{0*W*H{`CXxWS3OMZ~)rN!H$ES${C$N3X3y+!V$68lQMNtVlJ@&K8it=rTOWEVlK zMh2@@eHSCKgCj>*sp;w(M4hJM;xFH_zg)SX&u}&Uu*ssty?vM`;ljnOG_2u7>NAf0 zvcr^xT03UMtVMNK2van#$D#s)kly{6Tb;Q@9(A_h>GmR}iY5_Jnknh3wjSCRKNS?q z11026BG%$C5QX@hMgE7>1*sBY>VWS@`mewifB&O059b`~I8ILl)?q}tjzcpp(ThgT zI9SKr8=92na%JyOyX{JOvz>kn8s+_&sSA`B<1@t zi~Dugu_Gr3Na>ys`9QRM{?X<+{(3UOdQ)WccYMebihl81)_t({^lIJ=6xN60pZ(Gv zyf_2~n9(xPKjit-36Ude<25?a^+6(!7UQ9g-;*d_I0jEw(P8PHW(Q>%2`w7+?8lHY z%T_3aiBh=9LVSQ(w@_g<&iNbvDMU`&28NtXvO!e zN`7yQJ5Sx0Du3W!KbWUK#Nr~@rXy`rPvt4#WLRq-QVSGcFCEXR(lJ8dS+|8)2OJY7=1e z`54f6V@Xs+SPk*|ZxrgwbG9@D^&HJhZow57rK{|`{T!a|`UOzIW&wjF2DQ-vQH`0h-6Hr$dWQv<%o{(;dTO~hWmHrT*TbP(#mR@FW{+n|Oe^r}F=J$D7V zv5NJ?{_ivs7vVkXaAz>njoYLYk&$ly6WbBFdnk8($&}Tklzj~_ya_4F4*bE+c(X$j z`jVjrxKLW|9Vp~Pf!m8 z9;0<>kR27_avb#M-C`Q)hH~HkN%w%r#ABFCU!Afk%K`;z%u)t`Fw z(xy{lh`2SG#thkv{A)IH*r;p0RNr)k?i&8rJk5sL;)5r*+84PIK?WcCC`y(L?r?{! zp=wX>%aIkeMI(?rx*$T7l5Eiu_#ir{6D>z%Q+j_?$zp_h?PjL@@6&3oD)}(SOT|_0 z7_IUdW@DJ{_OUj;W9fBKr8!C3#V^w9`k8^sq_M5mJyG(xVu?0WY3bH;X00&2{cR-u zQy{wRnr{2-{8&dK-#x@4{cktk!6JCs43x!KOV{oC_JOv6s6o*;NjdEd-u!E_`VQvs zL`VE3DUjla@Yc@qJ;~2%qokdq)hFL8bA$4Yy~C&4Zo;nlc;iLO29o#O?8g%|KW#6q zIq@I!3eTy_g6gNtpFd$W`&B?S*YUO7Y_05SnllF&3R&%ZjCR@&I}WR(Febc0S^vaD zb+75A*tx>vJJc{a7gjVFos5p=49}vp{$|-`Riz_b;%T`i{r-TsodHITH40tlXBVa0 z(H9oV*Sl|%?hER5-j(VxE7%zqJ;SpHdaCAk$J%w9daE4+7)?edb~j8t|ETTux)9D9 zF&?TTJ%m`wH3~9CJ+WFCxTsEq&}Hs}*~*LWCJ}<$M(E2%fwi>1N=lHOlj=g769zlf zPXAP&hXl@bb>|62<^h4G_}HZ+PR#;!ZN}v~JFpeIUxl~yT3vj)W~_%O@fQF` ztvtMO^Aj^IoHRoSW?|z$Bf%8XXW#qYBPug&+2m3pePOm)Yq*ye_al`u{Hs04oqfoR zI<#+AW*iy;*N#M)KN1#-pPA@eIw|_%jUz6 z!hfZnuAy)J#^Uw6-GP{RV)jOs zaQ0@};ONPUKYAA}ga@ok?gQCd5Gm+n>T(PASEt;`>B9n$6j_bqp){5@2U7zjw1U^& zgy0V{ySLi*=))0hO?I5o_`sD57{Es0k;ttnE+1wUviA@SCjvpL0Cj79%4;K1?T5wcRE$n#I z1gCx~UHV=?%2rqDX$GMlHgWE7Aw}UW2p=ogNsy@sy#Pi@`)b%ZJ82DQ-GgF9R9&Nt z)2TWy3d&5V$dP27Hdndxy}n}hu?AD&Ewmq8<*1d=L4%qrwR-`-Aehfy-_%G^p$@F@ z*-k5YkId7pwcZ*plLEm>5MH=Di3BY1r;Kg79nz9jwuX6AU&YHrf@fWAvPxd}U$yrm{xiUReyEJL6wBBS2s%pNWRdPt zZxyoEA$;kGk1_Nqh$%DD#s^`cms!_S0 zZmxXH_k1Wh@w?X1LBmxD+ccg!@LRXxvI9EPlK!IlPspH#YIIjBL|{1GgfoYY1kFGd zfDASvp3Stp(Ug!oF!ZYyPN}oXUw`WkeAl@MNkW719CR)G(ne5jtf3xH(JyAgl&^f+ zE%&;7#l>~#Sag-)du5AwP~wpi_de2g=fv!p@cU==qvpKeLddTjN{{{y-*)IP5oTj#($bJroD#NY76T_Ag!?w z*^jtTQQ%d;jYY1}M=U2l1_Rdr67uKfbG(TI(L7|4^WKhqHR#1Xp4>1~7kxK|kj17Z zM9zyeV^WwmM4CQRSA%|698j&2wzjf3<)0?qJNkO;$zn`5F`}bae7!a^Y-b;nDa%xy zj#a1fSv|as%;*<jgg$^!eKxE!OB-|L+o=b0iThNNoQJ1T1wiJ(CS@P zV&sC#D?L`zb90BKZphDP5Unb3qbgWtvc!Lvdl-BNt7=9>P0OXZ&t+R)V{vAx!FlZ( z2Q9});0IVr*z8&PJW^syo5r-1RHxKY0qDAfMdBZIP5Wbbta8E@VHma+cpAKf^k><& zeOO({m+(X0vN!QjYWRxjnWumK`oOYlAo!e>*gT2R5}&9T_ua7L^d63?FyV*$6qBNA zDxW&yEeOZ_D&FK*5U!*mH6Ocv-y)DAnHbBr&cO&KzvOO4vi~~`5icdLyYk6fehB`@ zElRM1J=F<7Avlwh!)=BD7Nk`(N%turX4XT1|LM&By79~J_corN&X>2|J6F|bXXR&S z`6u0ZPwI#s;dRbXB%R?z>p8uyhHB03P&cud@$W6Yz&g_rZOHtb4@0DBQdfi2O{Z5F z-kU9o6=q#v=7iE5$CHfH3uy`mEKLv@s}s`5w*1!UD2=Jn9iE=R=y?`fQ~Ui2*fcXW zuJSC#^EGV%1v^348I1Z8fA|%nZ)rtg^tlT)ox&metRZKz2z}#dPS_sg1PH7l-`87$kocfBy zAUIWaxF4=@@FHnfkxmeBO1}%qOe!9XR~VtN3NGll*8p7A@Fva%qC4AHznETY|0SFp zPsg3&dlKr%&ktw|J6|;nsIIectdjR|{N+t#sGQG4;<>$MkN+bR2PvR2j=AvmcjsV` z46-leA8n=i4>PQivdZU{pw=;V4$5b7nn^{vdCw_b~k20)2J!&DgH{A^r8uZlUQrxaLGk14%Te!K; z8T7N;zMs?ebtt6fO;#>JtT6jEWgYu4 zS$jEexrM@XuHK)Y6H{O|z|Sz#2~l9iftqrmeY&S>`gSaAzjW)Hy$La9YE|FXCT5#Z znaX%dCZI=D>pb%y?%AOR;betLb*u=|I@v32q}4J+(Y5BQqnP0hr;rueDUnUIGqgiC z%x@l}j*-l?$qzSbZuBG(GblQxIh~)KbQTi?xPoXXpo`)3DXLIY#2~0$G1jIWEy^y; zGJ>7&aLXh)q?8Pt4;z|?8zLWRuBTprya*Tta&t8up-)$;lkapgl&qd9smGO^$eldN z$fn8cdkAfpHJq+JjM-4`U54gL&lH=rydkON>LXX}aP{kLy0-bfS!1Cs=C8j>13{Sx zg1e!*HJ>`vwjaZUa=Yj4O^ual&ZpuE={cr6+Vg1pBmX*A@}G06$e1qoxKww0~KPeJKc4NI$^rB5V%LyjTvUtMr!F|O%CfE;}J>dVTh zh91*oHT`-U{KmV>kG#O#hi(X5TAzp{s!4@}2)It7Cl0^WHgYMtBwiU|*l&(H3`n&s zg`tov+Z&pV>~;$?;GyU9TJ(CYH|aO?`@R3;c?@_Z$LBSaQt#ui;@3>K_g$OsRYSwC zZtiz5Zm*M2BTZrqOg~N@(m;x~W3I9r@2l7Mbc;hj{)8tB>RIsaT!vCj%mzVJgSyUz z;2KMY3z(KV$Lt=hT$!FXYfo3nZP4wS^K=Q4d8LjvV9hid19a*etzC!fHf7@2L89b5 z-;!c%r&4gbWrCX@#9oK*U3iMW++2eoO?Nv9 zT4B*aW5BAi(}xD2Aqkr3>UY=g%-^q@hU%O4#WJCOua)Uxa+PNR>q6&Q{&K8PvwT0& zPLviu|x7MTujuvqw4iV17rO+c&d84g{xY z3xtHIB{qP=);< zFe*i7MAn_nd*RAAVWeLyRT#_JOYGB5Drdb8SjnvtvL)No^yBHuNWy8rxrVs11y<4l z@-_2vA$$#YwL+e@7M!)2e6)W#8vCxatlMA|s43UC(yDc}cZi-X)T5p*20B;#v6fIz zU<#~d=^h3GG|$LqHj^twlrbBUmt*j|v8jKH%=puvMta1TsK=u#PVx@WCh!et%6S|> z%(Bo^dK&Jh{GC?VK(eFGj%j$TOl=cy6=e3V{+5)93Zr(jzKVT|Rmrkb0c&IH0zxGy zshsy*7#7*rdoOX7R(%m{vJ=*j+|qihZ~@p446Lk(!A6+XKuUXbm#L|nzKGzT?rYjf zAG=aLhZoY$$97VKL4y`Q(K6%~e}xv5hacX~+Z=7Z{0f9X`Ly7e&0*G0 zK_ynaQb8@Gzt?{tdMywCL^jq{QoNSC3MnVCliA*@@;YozL9o#RSM9F!zX39YwHs&0 z)rjO-t{-t+C3oFsG6Swh?WA#Er$+7CMIYwwGi(1r!=54Z zkY3>a)Tnw9;I~jz6{L5O0i~A7$Klac`tl5gICEqW55$nj4TM}{A79nn=_uhYkDk-x zD5xT9*|}%l^(AkEvAmm1Wl7jhlCDjqA{`Vqnh9fR_V3o#)NY)&MwA?IKwmYX$Rm>l zMPuJ+{daabn7C1{QD;{ufx;`5OyM4b6hI*4DJ&6# zU51i?{oCS}D&Sx6>B3jG;;Wg%>%N_p-DW;Gi~XI(g_yOLVq;!_J;HIB^V%0U=zc8Y!7NI;XTspRB2H4C9pe=AVtXq? z0{EGh06ZTTWJ1}lhpb}iEkzdd-#eIQ$Jvvw43rINBU73AoQu_Bgafn;K4ZL+JLQ?E zuO11}W3n|e9Ij90Zx%g6i4VuM4rAVlmro3dve`;X$LFT)a_{6vv~Bob*Jj3;Zu;gK z%4)eiUZKGSU;YR%$X%c?@KznS%}-ZVps*25Y_mAAnr7uT3)W9UsqD@{s5vIA3AK&ARVhrai_ZTn)C7HEmU)~-?oG_VCAuC? zbnMhog|=v2+pKk0Y`%1{P{NA8n=OoU6$VK`nJ3;UH51KS$>1c=ny71f#JVa{;QY+c z-WF;|cm3j!X0p@F3K`F}-KGa9&wD)IkQU}}*nGpB$ZRkn`1qf>YIf)(%w--)N!V#p zF?D&3=jLql9gfvs5!@DPxWxilqdLTRVRp7izYZR+sQqBQo|TE>yS(V)rn>?3hH-g| zPY9kUN`#V5dh>9F^}kL$@7@Cz_yN$)hKCA}4gpq*zP$<)}b8Mqr|)R7-`>* zJ%bsQ(~!YS^bw}?B8__TnZE@}yz)@kiCcd4iW{LmBjL&kqkhK_BS;<+4*ZIe@zEFj zo;6OsC&*pq7hukM*lFy&>}$k_0|D1DKN3cy0z)q5ER^=Q=)i(wq+d6{p~2J7g-u0b zyaZ;9(wc>+1JwK6QT61BgzF17d*6*}QI76sJ#j}h3c=?7Ev%dV;W&OjC z^tu#Te1uZ+X8*jXs2geTZYrL0}rgTG|~qmsdr; zj_&0NW6eDO2n`qSUSe8bMb`#A{Zt6{88w&3>ov3D1xs{Y>mIqYK!=%LJ6nc2RCKW^ zwPK4fhmV5-;x?)aAgz-_s4v$3gJZHoVvf27#A6ep2iya0)d?F}x{*c5;8-r16#t|@ zb*qp4z6mr}n)QH0wWShl$7##tiZU&_n053yx#WLQxHvqP2^#gNPo^ zp-^HK_&V8Yw7WJnj|r=F&r~weOz46GLLlYaQoy>`ZkB&rBQ5ce8ZcI{tbr5UNFGUW z73fxEo0S}G8}VI;Lp{%fXcgKKvbl!g`17$Ac{6GY*u_VnU1%HAjMVK7q&$}v-43ds zv$_41^3kF@hxFVn^(E6P@XZI^&EHUxdVX?$wvWqtCO@-_ZKzIao#cDFoYzHa)yn(; zZ%pz1UD?{OLgOt)HVz>nE~zdUr>Zqu)uyHAnD;hSzE=nHWxKz}dns<}-Whb>3#)h5 z(|>h0kc-r4YpSn%%}!x(vM`hLaXArW)J2V{c>t|Tm$H#^@wFBKYVA4EgCqn+uo*$6 z{kQMi19NTf$2rvovqQqY$k(Jms@ohZG*(1SZ9dF&y>D6Ig7PwyDNFa%7+(Tx19~L7 zk6!ujqVh8?|8_L(SqKwyWAx9&8N?!L2w9UuUw~DFOm#tv-4czT*532>e*vtZyXCdp z--#YJPWxc(n79cPM=n8zcLE&Z)a(b*(ZgjY(gVl;Zb)qaCFnNmrvD9S*??}tBTmf` zdt1Rp?G-aE2JJ}p$cxc&V5h`>sp=+uH$x2}V;m-sEueGA+gYMcM*HhEW;(KQC>?{|hZcCCk^ zhZ|h3G(G#@GLJ z8~4AoTn{13DdZxw?{zg|Ctg-#& zvF?{+BNBq6p_sC)13?jshU|z<1SKg!j3xBF%$yO0c_)k~DKxOY<_Y+T(8FP3**`QW zEm}1?znsc!VV$KAdUtflW~mMnB%$~2Y6Z{RSzr#3>&aOv#{{metYdR9KTIx?g25nd z!_=r(Bnv#836s9@%!)odL)TO`p?vb?_q42seLW$%7|UI<=l7gnlJ3hmoY-AR2}iC= zxr8sx_x;Sx4thF5&1bKP6wQ3nVuif#D7`FwN3WU4$XIz4+>A(VG6ZDHv>nYta&WBq zS!5{~;A1Q+kVL`K{=`_mWT&az&WpJy+Y$|F+WyVFjb2iVn&+yjktn$(P-oa#1q3?`5mxuchRvlMGRfv8cT7x`@BV>%7jU_Vw5FGsYTn zxXV_mzxsBFY6ry#n^bFLbUAct2YYQDXe1)6*%)e8gvO&j^sTtCd(uIp9m~cO4Bq$M z`{HSVF&&NCM^itJf$*^C<3m@s4l7aMBN$V7A?ZKarf3Nkm#Dm8&Oh)EI5MT>iVM&4 zF2nfaYH+F1|f4DdsUO68#t}Jug;fLyvE754k_C-yXrd zzQ)Y>MyrJ-y z5Is7W!DGmf#k$A>!iC?LH{-y+V=xYZAp>s;D?W+yy3CV)>)US6_Ucw%aU?3C`tc>m z*1OJTWopUSw!dX3S*Y93-&*Aq%D+Bt4JOjPo0#gVAOOB|15hO(o!eQ5WW6@21X zC#A{5VaEPh)owCFLjwUE9fd=|M9UL2p;9_jmUC2`CV63f4%_?fcs%UUh|1Pg={X!t zlu3V|-eWq#lfu;robIy&cwhN$qzW({zp{>o32PM=6O{32nBMT^B<`X1@I+QnCj;M? zwSp+k2|YBYp8FRT_bztd)$2ntXZTKbdcr}nkJ7f$ZnHl9qu&V@R1^0)L7e9i+~!+X zlSWM)aVVPP#A^xUCr=^K;@69a7s-9f1@%ou4~-{q%gg&;=LDvFshz{nUlS*YPpK?Z zty$wmA+k1j?NX*uUE^FP#j40DL~=~yjl3wr_b0+@6Lup4^1(D3dmuC62qR4#xGMYu^S#xgGV{oWCATPj1z?h~a417qw`O zbZ)?UuloAJhI5IhNLX=OB_@jGIOR!8FkS1!GY^Vbq9Ng=;{J(CmBHDseyOQ=4M3-%^Jr3+o!`=d$miHy%vNGsWM$ z`*P5cZIABg4dfN^6Kt__?AJ{%u^r`cVdu;IecBCY$NypM$%c zdE#XWOM(eNU!FbV{K$U2rlFGz3lIl8LG0nPS zR+ayfkZ=z8>(^#h0?6A)_bJ4MSkH6OXSDWRR z|DHzs=94WAkvpG0o!Up=6&{Dx+}5v>wmzBb(WBYIuYj8_@+k#Mqq3iniDPQ6aEKFO`spKe(7ZC1A zlHy7<(PT~qCoDOjin66N@1`-PIWzrrCay*5i|e_4SgPJ|AJZ%R#`zDUZPOV3W|F($ z7tV|17PEzj!c>lu__t}G)Dcmwcu#YbKc{ot5{J}L9yVTInBP1hfd~F7^%aNh+l{iB zAi1{~m{H?g;raeHe}d0E`Oe2vA}%?>lNiU;*!imIF~Hw;`Io_vEMC+9(@fLb+$5Jc zA%O9ec>Gvv{tkTWR~y;#yEVc9l$)`W{X8eJPpWq~BExE*gawry-ngPHfa(ME<5It! zT;IzaYS1>Ar{FkN$lJ+sR7AV0M$3I;uhs?anj7=_($=(tHL&PSKJ!*E-SSvfl<4EsX86@R|MwwxnrPzO1zu&d~2ZG`8jcOLr+qO}3(o zhz9>J-8H~54LrKv?xgN_deXjge4ZeH*{)Q*Y@f%}neUr}$R6*PoSuJ7*8)YL`6z`IIhPAk`~+Aeae za?{6Lfj??OinK~1e?<11S7M-+P+mONR!t2I#Zv-`G?oA7jNg!d+iTPjijBstdZ*Oqgq+-fp< zHd7q8f58PVL|=EnRtU;YB(bV(>GYls-+$V8!kC!`9LVVretws{kw5T%Q*w8bz?i{I ziF>+#MEbNoQa=lP3y=6+Z6Z`0TOIBc{61>!XGHibdw~%ix&`n^`uMd84?*X z2V<6`F9Tqel5d%hum5&7^eEFb^=Q~=wzC;Q)1lo2?%7Y3IiQhO-)^2++YRDbd?+kuy@|Tm6#q1!)nZbD{ZN$} z@V0KQ3iaZ6XCa#6F5{72QBnn_Og{!&7Iu$$xzEvua|$M9klC37 zF)bj9qP%5Y@Y7(yY-*LcKu98gvrrY&oLqvPf4Qq$&y_)u;U%NE)AfK{ z6V3$(g$dZ5Rn812!WlUoH1Xm&VW|~M+Jni60Avku!EPyQDj;Qle@F-RQh&nP4?i|a zbKWoIQz7M2Pco;gzt1U;IkTI1C)^&`%Cb0iZj5?Q`%CC8FXGRgxFd3$BT^a}{rGjH z^2Kg6I_&{F_L)LP50a2+Pm$c)xPW%C6BSi*(Ah$&mF;1>>nz+Kd5C6JqCaV!Lc;U2}$Dob2;~q0I0zFIWBR$rhV1J@fC~?T{17gS+ZEPa=(zDkht8% zL?rBoO#%FJ#bqmZ|K6y@0ztj%H6L}Xm)9>BhO}P-nWe^_-hrsy<8Vc4BC+nxbN@v3 zD=sq)g&7aiFRjfM{jHjyp^!bPp(nvu!l7h0W(n&7NQ_$SOKqtP02r*m(JYp~Yfpur zu?d^;ybP%eLXHwxQZ3UA9yU(L4_;nD-P^^S2CDi=N#{HOl)8vCP-Au=;q<`E$@s8L z`l*Q72{b#kN+>GgTJZ6Ruh<8dp~x5qFTPMz>dv27_p-ohat~!f!+m2~@7=%dqfQM2 zH2O#=6^ZS7zYI&#fbfH+Ov<$5A;FGZ~lnMW{o#E(Q-kJPw(E=6+K+32m9HLZBVJv&?KV6zimA# z*f?vzchW$t)dp3vJe-d}gcG6(r)p;2g88QkjOMxO&T9PfslGtyh=t=QdQ=cPq_@M#ybjK^L;3n~~G#p*@*B99=QjtJvsXk8?$)tfem( z(v(e-W#-@0fZX6NJf>ay1%XdzCiR^y&J3)0ST1hzSg57jm1@2HxT?a$l|PS6+ewOQ zW^0v#E^lX@b*md;`(}A8f38^}neY&7oCoVmSlh6!ICanS6-je7S9Zv3UU-V*a@cDk zO?!#Z$TV z0%!_0m3He# z<6C2Qg`~R=FhC%b_qU#2$PUzZ%<(kTk8%co9 z=n8;m`G`++@#NBy10xtxZDSycp{d$3L6?2xAQ|sS6kcb!nW?S3I7L}Yn%6X2a#dWOGen4t6K3; z7_A&EbCe{Sc{LI&)e*ArPy`(hlOKM4ytb1V!cmfg4C#0wfU=^EO-IJ^V6dN{x2_8*2c+`@s~LHR?a zgvX3BmlUR7^;y0~a$))itM!9>|D$?z%a&RRNE33SaxumbO8t1$wBc!p%Ct`Q^p;5E z^e>583G~_Iv)(7N7RYI*#NrG-7;9H?8gY-}pB{_VYyMZYs)=YUW-X>*wSmM0L7O9< zB31dpnZTg<#K8h3on%$Z-a*ns#Ugc>t8pElS_+PnBfU;Ev!l9-8&hPpvW^zcdK;>t zz3Y9lEe-G`&?P^6sJ{PS?bW-v#q#nhnv!#l1i^@tR$~G)Ov&$xXQ}>-!u>ZGk=UX! zb#W;O?upb^Q#coljeK#`KW6%iHc!uiOVV8Cvb8ke0JzE@5;oTS`^n7#hCXQ-^J*q^ zDGExj^155ajP>VYW-Cz@6eBSt8WosWBXtF(o|q;Ft6qDnT2L1T(q569qb(g9p_477 z5r%dxar=YnBAFr?`zhsvgo5hIQ-3vwE&BfW?~L5?pKsBr)0enZ-89j9;-lE9H(3&D z$~BDzN=d#5UV`0NXDJ6T){SRO&b#iai?BuV0pZosg`p7j9H?k%pcZK2cZph$=e}Qi zMv7?5HN?H0MWd%n=WsA=3s_*+PojQiD-^I;I*Ycka~Fu3EuKVKHqYTS^^6c6MU_`` z1^pU(#4QoA%4}CxpCkKXSL>0_%>CdvYQ=TDSH*8-5{s|g`GdQh zO29C3h~X94{=NM%mlm2db$REr`B?}N-b{?sJiZmBtbj+&?KrGVDBot;?5Y$5qU5Zo zSf=04CAZJ6K`NyV_Ms&jpA>+ybQNaQ`7A~9bH8X_6(v}oO@XnvXEHrHGmNa9tF7*C z-O1$2i_x|9LFF8SA0h@uK!P??H;hH?Gty>U0LtsYaO#N2_ENm$1R|?m6Y5d#F+=dS zffJHnzhs2QED_WRlb+6X)Yz@(S=yEU;Y%!Kl;P**Z_@@An@LkMEtX83Lw&fyrhEBH z)a{|-CLzCThPC62AGaVmt0YRSGv)Gh3tf}y&`0bszn%`x2@LnSst5_SAQe!E1A;bM zIkHSBT(U#+S!mLT1lxPU%8pG?cE*K%&BbFjt_{Cc<3Hcm=h&RuOuAY9V^vU^u0!7w zyIZYsR!`=X^;uJu>1^y+Oi2`X>62>j9yxC{I|ePgR#?N{9m5gqLx(ngxXPg{UT^<&0>3<whVR=+s*!c9ynOj72*qa`d#k}HeNKRKniBM%v zyHj22cQ+xea8?9_uV{rxME6}{pYTF4siikWTRwTiq!@Xi|5$bL-9aS1?Tu#&gdcfL zg=a>l1zYmnMH${u?B}Y@vxugu&9mjnWaCcQYY^_|iRNT=&-aS}5j|kKJe)s|9Y`D? zMfTKx$;6pkgUgavkiOwy`7A_LhOf8l+Rw3bQ;9F&y4=TwWz3oEuAeYNH_n+L+ZIgd zJL5z#1kIVu&WWZ1Dxw*JRB?46zu6grUesd)90;N-)Hj<9W;9TK;Gnb{a^zP_>(t@r zv_=z6T~(-2nUadTwxq@EO#fvh`m^uwj{|bpb};X^0_EKTR*Qi^3QFWB@+)n^l(75o#m^ba2S@0%C_yVe=1lyH`NwTg-Kt8Jh^Xl3Wooo*rYDAtjhr`BQAdUN< zks0Uj|DPmkrd<0!6Qpv!`#<}`bSdO2`hVU5)b_ux$fDYR%X`mL8N3kocuF6-k83O= zk4P1|pLgQTl;_CVrSm+yU4F~*mm((AxR%L_!w;Lva2ATzbNt1$U0Z2;4q(^DD6r#%ouW0;3fSr02R2ne z2^Veiv89e;I#g@cpz`QuPQ?lPe+qbk3WA0+I7zYa|0e^xD!BjaX#?kk1DRNA-w89e z4&?tP&C3RG4)OkOhi?W>Cg8=-|2}S$(_;Yv-h6%;YYv7DrWd|15K3tkz` zfkz9Py3?omwJJ1|SB$(HbY)y#`1m;}Nw7J`e-^gsnD*~r&qf78C5^;$=D@Z3TVM&i zXdj&HE_ZBs!h(hf%%0szwFv76TeeS4?6xmwdnj(Qzg3XyUIE4dTwMW*%n{IkSirCU z0#_jWgEKq;YAONvJ}B(k`2`K3D* zB2Y4P+=$s`s{R)k0T^D89s9T$@o6?qkBTxOAV(2P^S4khl45otd3mo{8yU+`LM&LF zAOs9q4$X0_O>Az>JhNr#u1JWyLa2vbVg|Fz5gL zj~~@Cl+28%QrvA2U$d<6D+%@%(gBE#K(!R;W+#d_7z$E~;z@Ufvu{l7MnDMS2_P$_ z$LnCy-9i#&{ZoZf?R(jRXpiRi8@lM{GU8sp%JJKgwitd)$jscjyg3r&rlT^Z2L6?9m(ZhGs4)&pG%&#Hx-)GPVPCG z@+ETu>9{`8zZO^yGC5{;ctwtZ)ZRrH36qhNs>B&2Sqp^Wvu;yp&SDy#fv_3AU0q=) z_x(;z8?kAbR-AaaRSkfW2XviGl{w3tgxb~VFXJXwxm?pMv%&%E?V3N%8+x6Z{c#&4 zS0aebdZ#?U0r7n654G$Y^gIr_^Le+Mp9}Ca_4UR-N5_CE$g#Bt z_-O{^xq0+6!|Ud`%d%6cRLD8%$6=@6Irlq{TuA5jp-)Nwt#$V&t6O5sDz2)AzT+}f z8qPD3Z-l-Mc7B)lxmd1v$zM<ZAFLStD=PP$=Wcq@>ZeSBrRRm>P$# zB*CG9pIzRdFC9@fB^z`g(t~$pNI=guvQcn|Gw4W&h4WZ-TVg_c5kPPB;Tk!P;{aEY zz3`>^T+a%TGfY-IjjB(86}wl8d;wCA^xm(_qeK7YVyIDCbiBT%GN1uUUJH&3BINT@ z6ynW*PI>!2W;QVoMCOC;9f^H4aiAR_#eUm#g#3jW!+ZenG8tBqKR{TvhL?j^(>+2-d+Y*p0{2S(~oQfY-4p zb>$E}mbZ*jg6v$Zy8P^N`{_E1l%5jbEL#F;?HRZMAq+&w8penV9^)n*U zqpy}y-x_RELL^texlPZqRC6E{EQv=m#i@i&n}j$-=_H&9R~?~585b@K9#wY!ap!Eo z=RTnUcwnK3VO!1ytsk$0sGNRpmvi1?jkNvMux=(h8Heq{bF^jy@yo}% ze_3Fkyd&Pl6>ANW*Op_~b!GrGA}HnwM~NF!v9-ea(eIy(Ag~*m_{_HMj~egoct0x& zQ_c6pwAcz2cK|fhku%s1GE~XoQ5&fSFlnvC-O$NlTJI}s)T{Y%3n59o%zBplL73~F z4|Mjty0qat9FF%MJ3uPPUhM7~o(wc>{@6sW4nhh^U7*?p7S6qQzbPsKtjEiF+QzfI z5g+s%vs(omHskI+iPTiP#Fj3VIO}vWGt1OOZvxj0hT) z83du#E|wZ32Fd3HHVBnqU;s2n1e9a?HD6Bv`Gpq_3qP4XJq-^3ra;K^Ql%k7P{_)erTZSV|tA`*s9 zLXFIU0r9e^!C>_C7)e=ujicm)B$xRO_OJupLFZ?&@9D?Q`w@4o_i-SAi0IcIvv#ob z-ggwE(k=w1jjJ@GRJbMrC+pt-PvJyz0Jd~j#s3|{vExutM?3X<%EQwETdeh|3ocxh zHN8L$*D#F4VuyCCbv&_}@#7_C8z_yFIC_hfdagTPN=TwzR(0Zpg8pHh662gA!2}9| zxupI40zt`MBgh~_N6(|K=s|mh8ffnI2wXEO!zXwTVNdG*RK?q_Yn<%v^LL0DOZgL}amA5rSA!X8R_etyMaAz^`N@X|2()*zOrC zU$9JuyTutObqu3fpG)>%h=G@Gw!G$qs3fE8nh5xB88@rs_E@0&g4b%q-v0c9j%5V1 z6>pJ6*(YM5DsipFKYMq&VyIReXUmrb;$fY2!nk*9e$hMUnuIeCG6vK;WkVS{I|s+a z*t_NY*eivlE%LJ_;wUTX8q z=_a5)&4pLCw5NeWUjl33ZKCG~+Uh=E*C*`Q%>`25RqaI1o69{MggjHzlyD_BMov=j zn*E-Kseno-h|3qPCBG&xHj-bw?h=NePZ5zT_3YvZ9S?>sffR5UOd(fKM6#>pSmmlm zq880>TdPhd!84Q)m!WI|j5SQ;^QjD^b`pU9=D;SRrgp5$2WZq&CE+OX)eM+C#f36z zzStkzZ@FM%af#G?PICrrA&I!0As)|O3&Dano8xLeYM>gzX!+i_)_|bO1xS6@c2yV# zmRED8L@6}@OHh)WGxPq+z{*RJuT$=lDVYQ}CPio68kh-WuaH+BX^6l^=A8^by|xv< zhVqoVi?*|_B}y%{3Q6G?S`mHp0t9*ycsgqsuaV7|&s(8c)GmWpE9e?yPwcf&c@Fw- zWs6{hqCR9+M{o*Z-mhru{Hm$T<6G%x3k524dRP<;sC^Wp>A2h9EAavsQrAdLK$}~NEPgXgu|dkEDGyeQ?Vp5agBO~@ zs6RtWOE1AjHjlM1t+R`XDj+r#)2LT1yyfYqQ``|r3WilXN~AvfQqI9bLPypHS9x`> zv;{EdZb~XMim|XjO<(^RQ1gIz$NbysN9?-xb`+Q7m?!g)c63&@l(&n?-;P`L7Vk!i zoecJGm?yD$K3t!Z0qHgPIxxSDNcB*VQ{0XsFXLIeM_n4iBpEr015cQ>u-Y~`g~;vb zAq(D`jXhZrxV@C!96!sxVVOc=c&J_YcUW22s1`&nB`SZQF>NSYyM2=*2&vA|8pMcq zdM;t5p^}4TEGf*fY2QdgHsN>zN?Wu=$c}J)Jtf>hbfK^F%lRHOF*TR?bG)0L{0Z8Y zeNBrh!Nv^;6N(kll62OO5QTaJh(M(44>>6!4Nh2)ZdcI+v0-8P9T#^{jfa~;5?1H3Ll0NC8Sa&C z{3fof^fQxF=lON!Tw20~qLuh#fEah6jE7T}Q9qo7U4X>S#?9vW?U&@Di64_ulN+~_ zSFp6MGGB5ffUqlwC383WE2_ABjH{_00f`PWge9e9czg8S(4x zC@s&Ai*3s}nNbAuq;6(M_xlAdu*gvLv454?pX0YfSm`6G?gGi&#I0*&ojP3hvRmel z{95bULtvCS1=>2^=PZYrgp{DzMfaRzoql2QNFl10RuLPnS^>&4(A0PuT!;lkS~-*B zsiIpd(LNPHZJ-w5b>*s!rzy>->K76)VvP^F4CeZf#!$$ig?k^lW8>=lzHU!FUgzP+ zaZ`D(ZPz-qI%1ov^9#n4Ot-haLVvcOYe4hF`|LtwN8>!RCt^E6JEcF0C3|zrC?RkO zhK9RHLs_;9M`9=#TqqjnBAW}%dz-pkWsRBSO7mm-pP7>k3u7F{R53hAy+mS~>A)Y) zdQY#1_w7BLlg5ADZ(_P6)PJ=)--i1(ea+T`O(Y;31uRkdE?eW96ylBowP~)Qd9T~NZf>I@K?_LW_v3aCs;Zy`sJfs!~S1;`Sd?SW$S-Ve!{Bx)1v=%mgfFn zfkknZYd_!F&}q?IxQ}xG@9d%z#T}Oe;=lC$eR8PhsQdTgLvciEG{K8J{?uE6QBDoH zc@ry(*i46rzau#AJL|bTGM(=IQWAPiw>VV&`A3ijZ-h-hKQkVT*6fO@zL}<8KxJPN zH1&#Kdu%h~5;}<%d?slLmeC3UPi#rv@s`o!Jr=QuGv=$>JM6=w<&ykLK3&-S^O~H3 z|AsS|3RfFl5_V{R{wjSdkjf1qn5P=xb)Qak-Z!+rVbyws0f{r0sg*7*+ZX9AmkDVkvr$DI`H=<4uS z-Ts-vg)OlLu%rW9F1FY6tnn9MDXKC|S$SE3?Vgs|fq&+HW#DPH?qJklG zz%B_PWXJkjYAW~W2iR;D&^N=|U{0`WE$jQEeIaFz))GnG)Rz5RNTI1zg8v5vvCF>v zQ?_Hi_iKutkyBHrCT?EV_gE&KQ~r^?shv5f`+rQyDBG)yRbsd!Ji+7aUQ^42hM18U zCE;(gCQq;=!j7EQ32m`O3W>jv5{wdHd!-s(l?3_3D|pIueK%Y^M=7)$ddJP(Ir`5y zwDlq{^HfV;{8U9)#Kj*RZR_-NNVjts-OTp7qv8)0aJcRn3OJ4j<{$ETMI@kOu}_|1 zT-M|4Q^Pu^h`vR>S`onFTVkTtO-8p%2e*rVw5=yV0w*je*8u}qwczBt7lvtE;2YOO z-M_;>zGD07+OLXy`VeG)iDRpvnaxOsa;zAXsnMitrg}|oN@tz!l08X_nK3lm@|ig| zgP?1Zm8LbM@F^Ipo^_|g&22M`Lfr8>GxZ!ihtHrJoQ5;E;*YYKbTo3la8zu~zX2%B zVrT8F=z&1PlHVH!)D00?2Gxo0iLEjXqdmyC5$=nnG+x2pl&*ZI6paVjG?T7NUVSmi zClof6i=;-a#|oY9r~DjN(;Ccfz?lXtG`7(P_ub*dL!Ct{|dp!LNbDkw$r36acojn_st1#tSU$jL$RZwbPgW6OcXDj#bV zbsYO+f^QU?*?O#8qy<$Hf~kUW5@ZtTACk>u@f&fL7;zqBfsVK^p>s@dnujzpIPi-) zXMMK4eIsiln2)7N23V#s81=3B#kY@0=bV$G^;Hk|d@Vg%_CoAGvr{4FD|KlBa>SDz z?i~6ijPaUXJr1utpEvQEdfF2z8k7Bbusx*}4Lu!X!HS`}h4EIshv3mo&?K;Ml_4gH zrt+C~SZ-L-MDM;5h{qlzfzGnBVnW>mwy1bav7CY#Fyc$(#-eGH!Uf4YO-5Fq1tFw_DkXTXMgu&0B)@OT6JcXx~0aCK2}o8jCi za!Dx{MoWnaQff^WIE~|F%kKGoUa@7D3Rk6@h?=&<$~|C_B~Lf5i!v4hRm@30qsw2y z`7?rKRU~XG{?L+vFS0r~Pq?V5MLw0q5l`zt@}WT(-;Nk zY3o(%dJh!4|ArN1y!X5oO7clJ*4`{M5W7G@3avVxwKhOE00eA? z16cu0t8_R%4z9+#7obR?+RP8XewK;AFO4&jlT~Ae`7i7wjUp3E*`duBJ?n|G6{d@}CxeQBuFp_OT-yV38vp+MZ|%2mX#$5c z68aybpLts;a@zLJ;D4?krNaL_J?#IB>ah7Q;#q)Yf8Iw_K(lQ)Tzbd$Kbu_(a@v&z zL7wN2tMmWH{Ad4voq{L#R)S54^b@oF5L=dAK_Y`jD^Yq~O#$IL^*NaE&V}O7l}b&P zi+wE(k}1!@f-|HmizSe}4^O8e5K>i!1k% zJON87DwsQ{!gW1DA8{&9Ycu);kYi>$3f^K#{OksDKDe1z*m_4H#PFsDrW57J$|Jaa zVFz0%)5p#RkE=>b3bu-|CW=)fPe}Dzi3b0k8`!qKW_CTGkScJ$f#|6_UH7xMePjBv zQ!BgxNc>DUWHsw~7C)0GqAB(Y{bj|*d{(@r z-Sk_i(Q-|UBk6%6qB&uA+f2bes+QGlR+ghM!|mf-BL&hegS^RQ?ChtZh-D&TbW@~r zb`b*+f-!dzF+QdZ|COZcY5ERj_-yU9$r@R)EhcjzTX~CDem^uB#1{x^}gpeY9Q4oGZWv) zs#W2S3ubO$dV{i1bZ-NxKtmw|@Hw&NRza1IIFMH;pY$~@0B!ODi9VJN(ONu9+h=ZD z6|q@^fPxQ*l<>Opc{hq0sYS4eo#f>XtJNlOB7F6X!fBJG`G1l7t!A7gxix*VKk27f z!d|otE-Ol{`+_5wZeoN&)p%!c{G#jKAYUaDM)VC?b9aViow}@6M;7O%3?47~?H7>r z#LsGxML?5heIe{n)^f`l`nLa_CHkUnqGzju`1xEM#Wblcl=`wJb_2q6lUG+lu6vVR z8xX8@6yyE?_T0Iq+x^?7AM#~pHMwf5D!Dc*{YSUxmIRG@u&xi|1dZF0c`aToc^^;3 z8{qgQrU6%&=#k(=TfX`WDq}{nF!ya~gGjH+fYu=pu4uW5q7C8a7%9%I~>w5oIB5@2Pve^ylX2&UT_W! zLjY;-3=1l8)@X^q){8+hM35r^s&}Zg2!@Lv_cC-I4fUZd%l{<7dqDZ@&v;b;zdg^S z0QMTf(K!*QPQF)6DjTWd2Vd+~`VPGO8%4*{Y~|=A*UD9DO3nuQ(-Z)T0VgelzsC{| zwcVGmx7nbb!$dO6NPX4^gxI@aLejKW67kkbmxH<_#(&NeNU6Y7vdprn?rK*eb`IZG z0BOIxcaVQpl^e|Y2@X}s`mh7SIeh{J6XHcP$bf5M)DD(My)1Yzvv1kI+=Ny*No1BN zbpngZ*^QbZ||vddC{1YDFrdx@`quFiz8haOyKA)0m!He(l#HQZfo&I1u# z9lf8WA|z8a^CPmuX*5vBB7lmeLfY}S`memYVfS%ce#4#QC1{g$QRFJWwV2GW}MNXnyI}?uQO$ zIEpz~VcrZllx!o)46L?Y8-uNLhCqF(g$3oCP3P4B0?Sf~Ys$P<7|0HFD)o1ddn4EY z>|LQx)y`wjb?)eeKF61wBhlBi!IPklD==Nw=s7R0Fu@&6-(>uW9 z@HflhE zwW;M)+ev<9Ep(gSi4k9S91P414tN5y=6U<`T2?1@q##asTJcZ7_SyzSWK8jzeeGYTgqYo2FMy^n%No_qhG)UGN z3__SVOdat1L~` zRnDGq9PM#y)ij0nktyHBSP)tmZv^$5{Z;0OS&wwBX{PCbr~3ImHr^pqKQC44ap{Rj zH*!vsbnL@7@UBP{4w3psh=@#XD{)e+lfniyInPCnU53@ZDG7gh*eS)H^?${x!IC7B zPtE<|hJUxqfp~L~DM_qQB;MDi(Q0>QyX~dnN&keJkv64EfW{(ZK|$E85uGyS!bc7C z?qf6EQriB>9D-yccefu)e_(6iB5xD*lQ5zNX>_xa(NyOc1w*+X#8yg9kV+)otNKv1 z_gTUv1-+kgk6-D3$@s&0>&W!NTWdm0Eu9UR#pB zTQ^DcVw=TXBvEh%bsg*ti(Z1;^lF2jV9#KG&_fi&Mvp}IJMY1TMwb;IbF;PW;GiCs zt0}b>Y+71Ou!Eeifg) zHf6CwM%O9+@~>s&ppBPuzL^L35Qa3{3LZRWWpUr&HN^=4>?yyNR@P}R0r0IdE5VH zMwOiVCuWfZvcyK7G$;-Q)ufyTJU?;M39`rSdwzaJEvnR!V>(#OoJAdT9ybo42TeW- zlgJdtJTY_qw>Id#=d2Rvzcax3%iVwloJ*2HkwY4J|Jrnt{DAldr#vwRpdbhq}6VAnLJ zlyqQ=-haT#vLck#ElRS}%}c&M5oS}3s~elN>}J(X%&Hqvv*#Knf&YnzfndT{~Z>-r_@bGe?Jt-;`6Dlt=azH zB&t7S-PcFUwxd;CJN_@8<-fr{{gMABi#6+ND@}enZi|vcH|+mb8KTKVzi&5-GY^H* z+XA6c*63U3CoPmC#F6}72hHyetA-yB(RK`!kf`pTjpEoz<|+9;zIIG^KUoPg2EmArzPs&~pU)_zlBPmhpDO=sGt*HZ@Gv=)aF;iXtE2u&!qPCyP$85nQOQ~AIH zx-$)=DK7tvQ4ZWq$uJjEP0y4k%!axOXseBsnd|!COav4BCoZ0f3Mr8cqv~n+wXem@ z!#ln1Atq0_^hm9S6BON4!5E@UIZk=IdLoM`OV9V>oAH^xhAhovoEk`;OB2y`23d)a zE?<-^<;9ao0Yx+Vp})7)H8}r#AQzp_+|3lXETA-*s}B4yF}Up4%-ltZI3NkyuXCpR zmmEUQ=mIb^O;+ zJ#k|LKK?{#Z71_5)nkj7PzrHC!)rTVfu=Zg zI$l(kB?=B9O^zrm0~nq${!4;ir{e$27l0{0_wxKH^0ok(m4g;i%VjMtS;pQ>m*t<$ zPK~>qr|AFrQS3_1fIn|)oMhV=kULfZMgxxn};xmclK zyehC$=+n$TlQ6$D(8EHm*H?M2t-^I|GsD+z?+$jmj}~I{A7e~lr4VB+q?`ax6OQNE!EHUz@O@S zyk*t~H+LXASy!IdZh|7$Zs9uHS}M{as}vJFBP!j8rF>y|@Qhx65U7>K?30aHnZ;+2 zvuF8|?V^EOSjES9O%@p^BI1=TRli888PUXH;hLD&_`ErpmME*(?syNuoh}N%O1OaH znOU!7^zIgWz`~Y`H7@_m@7ur3(YlWnd^P-1x|ccMNe)}*J<|C1Nnmo?*Ovrmw(43! zWq>2~#?AT*%vB(*wa;=(DXUeuE`Tf=ArJrRh&QVrCig(BzE;p`>Qe2Ts73QNs5j`9 zhjDjvp3ou%2uM*8YcxAcLT!mcy$Yznt4m1zCa^@wh=>xvR;QWc3&5tbqJk)Aud)dw zN@ad9emlHq)vuY&!m#quX;gKxeVS)Tn{*T0(ang_l3tS5X_S9S=>bcuR94g9juRxC z*EFGfdT65wavicC4Cro5=ev2xGO}fNa(S`R=3vx+_k|=WM||A{!popQl&x0*f+A?& zbbVpiJ_-odtm!Rj%Y?EpH6jvkoAQBIw{;5Y&F1(gLa?hki4-Mk7u&*-nnhUERbw6 zB)P?9$uLRY)0$Q&#L`Zxs|5oodU1WdexbFqt#V(h_AQ0XenL!Cfnmfv3hfbLrb9)S zm0fmE_FG!`7#fI->DgbBd3DZ)|M&mOhp5ylWg{CXLezw#uu{<|2~&p*foW?-Ng}Eo ztX6q>nHG0JWgx6oE2tWqha3IntjZ^|!Z6};Mr)2w`=v)n zePOHii1BQFpyH<6^(mCmX&#DCFS84~jfz%WXZbLYkjLCgI%K+=w57vz^NP)t7QBW2 z8%)nKh5u%rrkFwD(oTd4RGCTZ8_jzyr5E&7VcF?;ZIN3NKHCe;`_1yK+j@+LN9j)m z-P*SyaKsJ3#<%wL!VODLcsF!nL|;3=W&L?E2%SKEX}Y&#c(us#K@L_iTL*@@>>32* z9D*?582n*yNm~6Ni_BD;7~l#HR%p0;WRg3}R+`ZR>hge1{7H@Uiv&6WB`U_$3U(m8 zPs%D^xN7vxtMOnBq0xE!Uvkh=cnb@7Mx%yP@Z9~%3OR~nc$Z!{Z+qx&Q4;Yc& zF=$supSPx_tDB`p?V`oL0Ez_MeSwuZcW`<7r6vSMHn5KbCfPR2A@;NJV3Yy#q`;3N zy8zL^R7%h7GKWlPX1cckn@eaW{Le8(O}4K*`!0EliahX?CeY%n@!5n4$NC>7Ufyh( z2T|Fncyj8bBmq49LNf*iB#fw9&jjDS>|I;kxi0;)=aUhYXxNs+Ke8Cw68;4`s6r<= z5(&Ut|G0)E#Y#u{2-x2;k97{Ve1V^zmEGjUA3Xu?IsQD(ACuCMW?*LU(J2Kxy^$a% zI=X_qrpo^jTB!R3#!I*#!84}ty{Z*z*=%dzU%(zs?O1hu?mNxZtdXD`MQOS(_FJeqb&p!Uw($M1S7vFHP?6d*_@0$WNI z6@adu+nd`!PQ&6Qdxq!2SE!T6Z}t^@59D}sorCayD(brKCd0Lk1A2Xy;nCr{PQGh5 z1rs5Blfgd)Kqmw7j{N;vM3@PuzCXKq^R6L1zW(^yO`Uy`Nw2GL6Sa-1A4nV13%(Fc zEASdE=yAT6q2hhHr1fpUI_ zJK+YNDx2Jbpr3f$}?)|&gc_kLpo zZ0ba{?UR-eGN{(X-s10W=xJD0S`cp2&9X^?iBCcaKqb`fP%O0oHE&AV+1b!@eGYi*IR#^x zN+I18FRU8JmPrV5gKC806qhjs&(B@9W3v?1naK~ZRKf&z)od#6ANq2~MHAwJExRf@ zi@hYt4MEx2P>e27ZfZ8mV(l!o8f*GSXY5Z~ekG7FMwWK1J1UIAq=Cjt=K9LdmR}pQ z9J zaw!qZ)pn1o*zU+W(l%8)(RGf7Sx06rn{q7CP_BtTSA6D-{z5~X=$C?Fz5sj*K-d`1 ze>@Pn>@?d(CfIXjegj689OsvR5*O$pl2z>LsUdcMseR2P=e*F|)uHs)ZlJ0hE0@LG zZ3BVv=z80~MR1ue(pnj|!!D*)tn_#M$*^OZNSH($BnIVS6qMXSu5LHvK^mE?3MyLH z^534*?8_;n?NnCAD3gm)YwOIvLIi2Lgjj@b@CKyfO3tB#ws|X4yZo_!XT~FpDd1-D zG2BBOd6@cWxtQf#6bDw@fliXeP;qfuwZ}KDN-*h?*z% z{wHVU&HD$XtS+a($+GfgKt8=@9-7|=tW|!z-u?e^#ZXYQmAt0oJ~B0}zG7ubeEFDl zcC}v)ugCk3vhg?%i3s))ShlmhNuKSZ={OH4mj@_JoB4iv4^9!3wzE4M`1Hs#@Sc)( zP7ww_W@bxmLnMyVl9%@bl;D~o&R|%sa~MV26{|=I?^{Y}Ufue~ICVlH+Oe@rR-VGj zdwe62%g)RVz6~{qxPaKFx9*mz?*%9~B#^Y(rk`a#y&u5&<#>O1NtX~589gpD(FAG- z0j_NSUpt{@R@z!At>;QN6JI;}Wqnq=f;!AIj+#n+k~t;Ks35SRysZD}|6h`LK_(KJ za)|M8*N4R2ZZ*q7)~qO^!s+kX|1e^w%rJJB7z3@x8mY-zwqcc8ko4jgO&7Azd#}cJ z;{Hk$E#EJ={oDqpssk7Y%vu0rb)ha2%JSM^K)Kus#B%9I!@AzA`7beMdi0m=eEnK@ zwr24}c+gR8oHlU>LFN?*B59JaWno-VgGucufS&Psyk@Y0=tVwCP$!XsPS6%hHX@#C z^jPsJ_2g`^d=4Lrsa{?908(aK#0B7NXDYokIy6}SaNBZH)D0mrqa!Fm`T*psm|9el!9rA0ubGfVkxn|0qIgQtXX>)MK};Tq_}-6G$i5Yw&NFBXD9$gcwYoGEJdrCtZXIZuGyR^C1{xFRy zX7UZF5)eo7zG`=P-r$oaIDB89kJ$EuxsX~OrnE9Z$~FCv!e4_VecBsDC{rzS?nu_X zP6t9HFhq{lMMt{@c@YtPUKcXW!8JaiM{)xBN{{D9R@~iWpZF(#n`W70G~S%4R|67y z3hL8Sdtr0TunVUcWe6{DDFU(ZwnF%-eWv>w11*afvHTw>wN|}@#9cKgSWQgE|LRtj zY?n^l<*G~N3YvH-eV0~G8-@Lk4xUyjmWU574kk0Ul!{Vy;@S{AWBlt{qPy?|2uD0m zCGS|z`qDnvY=#4*Y4O)x7pPrc2eg{z-A?rIpb-p zKQnY3fNR8oRC=_+upX8=x^q&G7!?%3qVwi-YP;%yjm|Du_BG4yw+%1ocz|>|ea;Vg z0(;L`HXCE@_l*yAId$EFNj=BYZkm!CuR!C_$8Ii$?T*FPNSdmIGY`Sm6{LUu;yN$^ zfYVQ8-r0Q6nkUhFgMLwd2|R6w@z~-_Vt!C_{^O#l=&ZD+*t+-pc#|b)xh!tM+t8ep zbY;I<5lSehj#dZCA|z8=e&2fD)dd36AO%V-9?GCP&Tdl{)7`%w{(SFjq+F#$VIl(W zH8>=rfoH#K}KIc@>vEnxnFBnNNcF21(3S z){3grNdt?jUK3^)>JEfeWKxkTP;Z4uwSxcm$1>0|-mx%6acMP{B53d{1>-{O&y@smY)tI+s1vneFMW}c2-Xtn|S@YC! zn8v-I{~he(GS|^Qg@is8i7d)xcG${4)O90|;uAQD<2D+7Zy`r`_~9a{zFL5tnxuB$ zCeMBJw=f3b6UWNaLHI95N-CI&+g$Pfw5M zpp~(~odkA`Uf4f3=2(a8S70dewV3p?=wkBh8i}&23yU>5P7M;(@!zeB4v{LP{M4aR zMY&@YMyGnA^J+9`F6TIf!&IE|Gfa`e+YnDLQ^G|?_KNqD{eg&wCcko^3+sFmizziKK;$nsT9Q=>1}Jrv-lWAS&!t|qs#gA*+}W?7&tFfv;7m$d z6Qpu-^k^acCc-HX6-pbPK5C&>O+x+bNOG6yIs2oN=4ZlcmJ&!eZr+Sfp3!05NyTVdyWs8Q30Q4=k4$M}gWrPOR`#+&_u8p&%X-C=b_O#@pn`+PTrJQS zO1@$P;7F6dQjEucg1?(Ob1&ixjl&)fU+GCpT*n8RHSY>EWV4o_kynEvq(xJ+L#j!L z8g5+Y+%@t(2$z?D`Cc-z)+1WbM9Z9%k_% zzzVx0f-plO<5;8$Il?kitmcMyJniDPBpa9jbjJfU7Y3WjAuZq!w^c-0H^{2i^DirH zA(>^D*YE9mgc6s>%;I3Emo2K!Gi>{NzxELIz6=5Y&B^72X_$8OUk;yJuoAIS8B1QH zszUBgC~l{k0((pZ+~Z?-;;*tX0Sk9>91Btm=a9NH6*4Ar;1=odB%SJV&W17j`WAx0>{=!m5MCZe+}325Dqh)C5%nkXJFgw$JkPg{;&ma>5< z9jTp~Pwe;W%j*CBetSP0P95o*SIauv5}|A3rl~V=Wl>o2s7TW?odEQ-3B!1jZ+H3Z zgbMv04Y$m=$FU-Kt7@pVkbMYEX_3~|p1(MbRpnBzF$bh|?KUKf1jeJGr#hj{4@!xT zTwT1GD>B(g%(fiHB+RHLW9s7%b5D?~lhWj5%yNi-@4s!IX`MIzQu#z#3QxWCtkjcA z-|%Cx7+@rYYZs70C0Su`m?qy+RlVp)?H_DA@$G{Y&qh2Z&Dr%akJC!zs8_4-Po}hz zP3%!4L>Vt-STe@3dPbG0px8Atms(s2$YVp#I?HXMd$3H!u&M1F0^{%d2N%D)Az9Tvm2@V;_UGHrqRfTIUxrN zvU>0l5EM)U8GyD8cFhe0Lf%S}bQ#NWz?d~_$EmB&kE4m!=}NX*KjvwV70CKYL~Iu0 zk8x6+&l8QL?Est_EtMnCI4LPl@qvLM!*1;y*}u_8scha8=Jj-1WLZ}f_wTfb`>p&3 zGXAgmPvI_RNunIH1nQk*?X!J+bQQAmdR1F z5aXt9v&1+>S*A5N!0jz^OxF2(=?fYYm2ss!#~_6!EJe1bg=sYDUu*5dYYucJ91XB% zbu+*2<3PKt8r@nLn1L;BKQFCYjC2Kc6bPPD@&PA^o0w!wT8@ku98cJk*-Tm^xPMj} zpDf}JlOa|GgRXuUnN;#KFLzO=vvoT<-8b%(=&2!&4m7Y>z>yqhgD*r_ zmXYZzG@v?5e6WCDnlb5Ps+^JZcSL*Rn9;;wiWCttieR!7(dWQX)kp1k6&hSregf^p?Z_GrkBph=; z#VhC6*kX5fVVc~N5cxRM+1Kx#>oj7=k{A4>nMr8=(X%h{^y@ZShx9ro1xI%{f`h%U zVT+z&&;~Aov9cGeuOY~1GC9^TnHwVh@}gN_!?V9W}Q`L}U1dAfoh-31T?LJ9r|j8XX344hGT zb!MSeq|m)g2i~y;ti~(c`hltdIv3N~ukL=t`40~Y@^OU?^rCDdcGEeJZ{liB-sA?e z#=!L5X6FvW56;}f`VR)$ZBkD1Z=6UDfjq+9l5KCDxo90cXPC8`If(b>umr8szy^ux z@NIN@a2S*TRau3LKi?Ombp~<}n<(2z^l{~=k%uj{=sFWp`_Rb6=9b=GKTNT*g3MyD zD{}^#-Z{}6E}tpKogw>2D9}+#HFFQGtFZ`d7>@k7tW4hM3d0y%)tq<4Z+D{MU)(ac zCKpEI%(5IbSm?6Yc4ho^u8kGeJebph>&sC`!?X?6()PfAAwWywR$4K7S#a_9TZV;e zsS$&Es}JdB=H4vJkfg97B0!dV4#0&Y+x4wr3PTJM+sJS6P7zSK@~=k^^)+0XCG5CV z(H2>(G3PSkMTIGqNZh-lu;-RaDc36Ju7qarJvn#Q_&B3jbuPR5bXT>wiOCXakm4t!}0iowIXN~?NwMflVgr%I}YV?>34e_ zo__)ErJ~a5Nt$wgdp+bMO{9fzAO$WVRWM0Cw@d+``V7F>m3Z4#bmlT^JjDN2YqBWL zb8p1cZ6(O4swhE-vTdCKLd$LDz@=f&uYGFbhB=}L!edZbD$vTKZDHD;Tn<0ym ziP1(N$3h^O{3oa1N|a^?*^G+jBg=diq-H9(nWa!I-54Xi`(w7T2@8qV-;^bKVo?e$ zeZg0ly4-;2-oX>=q?QZK)M12~2lgeRhVM=KVchBE+=YY{yu3vmjb4`h{TZ}^6NdO3 zn@jarmeo<{LzbCE$H^up_JB@)y3DJP%~V+c_Y}x)m|>_|MfTm#gDqw%Rd1WaRSI`(>^3#=i3 zXcD>Ekn3PVA#(-WKtm+w-6OeKx|s`SJ+zRieKFw9z<3{q9%&B!P8-V(+SNd9VN&J< zVwBMq<6S+r5lfcVe)j;d=hNL=NRgi>KABT+dT0A>v<`r+;N=Hyu4{Ji5S|l>TpU2o@@R3)3Z#BVi$8b$H)F+^Xn-zR?JF3~ zCK}Er7_7GBd8W!f*XxcMf$y`0@7GCF6G86u_vLjj&wIT;VlUtK{VT=$==Jo>%(U;z z#(CY2UiTZ1!1qIt`CpEgm9i#+)v8_!6fS}9`(Cgt7ep(K=i)C2(@EI-Sub~KfxuGH z)<)!?y{ZNS@Yk@P>cjq|=pD9^SKv$zZR4(-)fwkxa*lMTSUSIns8t>v=X15W*^3>f zu4uZ=7Y4h<2g_RU=?q@bN!;eMVYs-eNNb^!Sc}R@W;NUUd}yQKGh|ZCHe!eK5+`R@Lav9c>37IH>x`ZvW9( z4ylm($CcCrt6#!aE0)#I!{X}@Ep&d@TwvMsw)Th=P(VIXcAORkso1(HsIR=37PfW8 z$XlVs;lZ0nX#){k0zIF>c<6&XW2 zlVrGm?kKPKEsf~@#yK!?JR#eo=FVQIm0%z1UaJ=562qKl-8pJT>Qmkule|!S_ zz$|e{lSKkl>}*H|I6=sUDlZbG*tP3$)Cd`sdKAl=ir2drQq=6@s1v`-^{=}Rb@;v| zNMwn*XdxfuUM0XKgN!)6gFui_ti~w;t_rGg@8zSktqTsljsI-;EiIU+JxW*L=<$i& zw`@^O)lGY%vw11=_5gokhX%f4AM1OW%Ha6As`xE0*qXgk87)A9NOu)oj8K4Csbrz_ zp{9kVu%K?;v0ztPQ!Ws?q_l(;`h2zgR?qp#N?DkcPFU`yT_)96$bSfS z_YeIhMOFn}$Ais{S*u^>taT}m4NAfyuWHb${bOC(q7dZBs=Y{jgE;i*wlW zFD|bBas40H|G0Sh`tl8VBWq?$Z8~WqbN1bkOr$U87+6TP0J{~$=!;z5sh}$ZIfIYr)#!X$p7uqav%E7tF_+N+Nii=hdCvh!uHIf>{@+vH zOw6KuID=$*rh#6Wlg9O(!7ezC?ERCZ-N}+3nES2~T?Dt52HDD-Eyz_;5LWj|U%;*y z(T|IogUaH`8vIBZ$-x@g}dA8A%)iBF%?Na*96{C67msZkx zgF4ujsvDN$7e>KzZa`4-Z4=+ewx9L*f!Krb;^0TrdTroLN)e9kw_2ipewx1(#qOPy z^;Zs0BYEX~j}mFm4GUrt1=@44?dVMv9=;aJ!(|}op&H+AKidGtwp-PR3-xs+o8Sqw zCt!Vj`BJYFErEhUBuI9R0ecQA}kC}UoHcNb)H;(``J5wFULqYl-%mzOJ{jnH?dRS&T%@tu-St80WIHR%RpOEbteKVP-7CrzOluPjKUuK4~?v+b2+>Kc_x5k+XNNPx`R#_6bhR&!X!Om54oN@`Ve z(ivbLZ5?Qc8!<8X77nN_St8V zVKdGHvQL@O0&vIUlS%p$+JCrB&O^@`e3ufTPEcZ|V7jmdLm_kKxlS?e_dOHDG- zZC;^1&8*il?@+I$0S7g}0a20^27n20xtFxw%t*xw%vdx#@&IOc^$Lh^q2EK5FIYOmCZ-i?#CA6&mr!`&RoQ zeZ{UySD9sBrjHy0ejayNpBU2_qHf_sG~J*5KrFN<^$^G5daLM*N_V5?w74$A@Z@9< zUibC&MfY*gc1)fNgWn0ww_K@M*oD6C*m5J~4LJ*z#+fU=R!_`gpX(mPfw8$LTu>8e zx}tWeInc<=^#UdWYe>Yo#m-Ua4g=PW&{O?jC(Zs;#dB+;)AhzEs6CwBXvYV~3}xGg z0~bX>D`RRJxaeQo=Sa>lMwCIE5#5VXOF0j$)tj6GWRD?z>KjNdib9VZg}~mpF5~%{ z+sY#jz!D>EwS?vMBuW$c)>#WiZ$qMMX)Bv-`A6N#DO7u5r$UOkKfLYF=L>QvTxv*| z%@VYI`h;xIRjoOpUq1)NhZZW1_iT8nmP)H8aTUf4&cru5Pu^}|d^Qj2bHpC9gU0A+ z|MrUC4H~5NEnTB@iXzfATE`62DHMlHfcLNflE<|Ik|9gr&t_nV-r@%iCkFNUp~igQ zv3Di&y$IydP8gg=2PNQ%bwV1Y$^%?9^*{1RV85|{V9;~7B`p|Gi`S%Pw&4N!*VdMC z@z?(e?860_T7>R7?>|MtyJI5+Jq3c*>=U8ROT*oRq$0|GH_Z+M2goyrX!K05= zcJ{Ahm7QaioxST=WhZd``ox=Km7SsMSY_u}W#?FB=U8RuSY_u}W#?FBXW*V-TvN_R zG44{JLQL>~oAFAP&1w}`Bbp(?Tlaz$S`@Ip4~f+?KqDESdkAPBA!|}{0ZXo83V<~P zdG+=yw^GJ6e~V!f9N$n3BedzQ7fCG_js1VF5U%ZBJvn6AK z$dVa{8YPO0b-@U&YnpAwS;hAdAddswMv9tmR9!K;#euHamU4Urp;;vZA+Fn|sCnsL zP85S6bvz^8z(FKR8&*2QUqVx6x+g^+S9AtY*W0q*VPu?v!mLzHvVyaKeI8qgSbFe& zhkh(fkCo}N$Uzb>Yej2F;bX6-P+h(}d3WJthM^?|viMWMBXS@wv_u*2+?F1t>_%LS{E4 zH$T=voR_wE+5=a$^jp{o{n*m7 zzdI?_DBC#-+!PGvcz_a=xB&n%G;L2`$=o~yn-*bK2ZXe=IQ(Q5JY`dV*Y>Hqr}Z~( zQ{!EMsebsa3=NpLlVJ~=rgTr(j*xyE>kR?BpT#XOQuo5(R_Hk}gvsgi(+Anl*x5>! zbRDfL(Cc&mQ9q?t3xDuCn7<<(E)5Nqw*UYDNr8?zJCL@jPqbD6A$N2~XeAqwFVI5Z z%r&ojjea8qbF~yBRJ+wS!4S&@dA4}=^p{|*(3&1=edAaw?wbc`m&Nj((6-mOa6PDA zZoxj`E3E{wwp3SS+|NKH(o&_DdwM>Epq=LZ;p z-{q4Na)x)FVzN~v5I%9bP4{1YEF*wky=1wPYF!>Bn9x{6#hC8-G(Hx8VHv&Z?*8JS|+2R%Q)6K$V#E4EDywX`CX*~hGC6u*54{*WR=xfAsl6roH`<4Zoes)q~d zpQ`~vYF*J_SGvfT@TPVgjTgOUm7o>?v!onzBi?VQkm_Ai49k`mZZ0cU7CfWo`5$X| z?jMzY@E_*4A6sB9^!!jL#X<(-EG;E4&$Pa87hsd0D%7SRF}7Vxc_p>ZH$c&>7Fw{X z9x!JH=B)rl4dBA|hrTylh%|O+IIXT%+7Mts4t+xC7 zWlsK64Ma}`7JDJr-wcDn?sX3lUw@9?e%7-BSvC@BHV2(#&-&F5{{i6FEnH#YDDgt<28fIM4# zF%%JiKE7v*FMe-EZZB8`cJOsgYqn~NYgWJRPSya;PL#^t@8KAQP-KFTIlE>BobNx! z0rV1*N6omrkv6#LO6C|arm_von*!X$llU}#d0oqjuGtIN!I3rpcK?ZX>nlT$DEwWY zAjl~OkHP{@PzwS&K=~_PF!WIu@(zMVb8y&lS@C)RZy;>4xnUz(>0bKR4coOj%vr@u zBB!2kSkBhQvK$9ThiVP>=^Hizjt~*2#+CeD=#dvmYm}0DGOmieWz{WbcTbE2JJ(ET zjuWb$KsMi#-@rd3U4H-ec*NG2iXd& zppxnqMFt33PCIf=&dNgWzy!dRp#soV)L_4`k9sE!okLPTU9jQeEjIfRQe=xAShI20 zO%qg0nZGvBYmv^KT(!6Q+#6LY4FAyzA;o@;Sz-)!xHZQbe}O+_rPMqBg8;7D76Av# z|A?)(Vvqoe0jn)qlnLw2nRrfqOFo>?vRrhznPyENhF}$-S2G)a_2C3HtvM(cA5In{ z}?AabsK~J1e%XNOHl3J(h@l1XuOUI z>aUoRO_dpF-!0u)0LtaAH8Ipu7A9&>pXuY#B^9eH&Tbi@)soj0t#%HL5dt-KuxIs_ z`e(zc9au3T|15-Jics)oS3=6dR_1)wVN`u#V_4Q`6szXzhUe^wH;Ew;VD!`(v;L6B zo&VRg1U(P#*VB#M66V|l5O$INMqVqcH5L3PX1H9<*0R!no~cZhW==rfJoko}?$CQP zLQ@%y1R|Y`1*!IGcNZ3Z)!_1|gwXM%xx~3RXc{L;xK>JQ1pjB_#btm{T2jrgE?x1= z(S4^EFlgg?2JQA!GeQfkbFVi<$t`PtfISJgC8M&wiG8hQi)^!DWX*(uLUycoR>DW+|rt7v?xZ=(=4=dS@A8cc(HSlkeJU7J&}U&oNY^4YiS4$W%2i8j|t9lqb*PKPq$sJpUB_G7^(nAXS(SI};>ZYSh z)1dm8@CshFlVazQuf?d;bGOmTTRXSC8N~uC;BCOl$Xc4o2In~zoMWdb$$jUWPsaL< z9O>X_(#|4EZWNBqW40{pu(PcW{s@g6HE&LK>w#dJWQsIa(VpF50OJLGNkG?WHiL`G z%h)o8AWUifzx1Q8L=FJ!HmJv{@n*eiMpBNP5t_!2@csoh_DAyKn@d8gb%Q?sx0+u$ zle_icL9=h4E}lJGJRMYnar4ioI+tS!wW&O^Y6lWD- z?VhjXP^__u*OG?2*aHiAf!+D@IsOPKXs|Y0-}e4y7(M6{q~;-ZYV^R^c)anz#yr=% zW0$%!X#adjKbljrBCUJplrcL|xWSF-#4NcjWcfg9E~JAF4B{2oBTYPxMFjvGuF>4o z6G&2Nldgv9RD`)}R!}|`*W9(jLIk@4QMm=rHQKvVOWW#cV`_iNvW0yBNzP17%!+ww z7M(JYXp-SW)N(M4g4@Qu*!LI;?3w2`uWyKQQ8RbQep1*G|4 z$q;ks+p1y1*t)@R^TFi?gXu^4NnuK@*wQ3==sS0hX8ol^lgDCu_l4GxWQk5YOPMbQ z)=1zzu1L(4%v}{Oe~wJ$wAw}NK;A4;57KRA9Fw+^6p5b{To?;23eb)*`KKR5v%SH;KAru*Uj03h z$ocu7|7D~>2VpjKrOt2g$Y}FY^xoOhj}RP0F`Y)Pph!(qd#1&)slbX`ljYWqB4qqm zbN8~=QKOz5WU93%|@4QZkjN5 ztb9(JEAV5*Y9@wrt(?{Si{J0tExW2bRK&F=65rFSHWc7aAWaOy0amgKLCSDNPQUp5 z<`l!Sdh$ob)=fbxJ=GP$ppYNUX4$Gw>eKeq=2QOZ?$hr4#{tr3TV530hEUpnw_G$e zB#AYZB01eDa>v+>B0EMaHIT*P?fR#m>}EaRwHx>R&ikBupEus;t@l~^->dUKpYQMQ z05_^+Q?sjzt@y`?n)6-{p}w1xaC>K_Dz@ed3WEVk_J)eQ7_K6)!mNC*EZn|P>At-* zCR*dlm4Z;Bn&rHtL-D$yF8QXv@j%F7d!}-&C=iBDGZ^qBAbQL#GdUA};ol+KhgJ(5 zkbG-OrfM&q!9*njipY5WURXqjRsPB?>=XsB{|I8TS*XT9I_lwGai zUA%cYmambn@>~R(gUxooXnsB_svACN!1&aRP;$d|GZRuoY-o&C846QQkPVuK?A3bM z@1tFfT0IAbjaqW^-7r@SA~#^E=suh(Q+Hnq zwc%w9RFHyqORntbyRz^|9@pQcn2|TK*8jZv$WgC`4%IY{$I_WVG;d*n8UUp7vuI*efdwA3^AX$rmLm>e>VJ< z7A7u+3c^r;A0aANd#B}$+-;;nAC$8^o!qe9$*i+O#!Z+fm*T`h+;umlJEuU3IDy|z z7JCQmm;xIHr};ay5SA!=?Vawe7Y{ucy^s!v0x~RJqK_W<1XzVLS>;@;j}p~U zq8c5JqeKPQQKA~XjuI6)N>o$Zca*4(64irARElK{MCM9SvyZWS|LgAEWFw1Q5r9mG zG0CP^)W1s-@y_Ei=+P8uK_W}2R;b$3|FqagIMsdFJHQV5dG{|*D!443%pCQ zdd+jP|F9`ULWF-N1>9WP*kg;0&VF1mroT-=f^84VsfeZ%&85Y=*C zS-IPtV2qr+74M|1UvmY=<_X+?asjwICu5@69`iL^9O~CR*kqw z=J=YkXvSAd{Tzm%i@MrDMC2actSb#P(x-0~Q?3StJ-}RX7szb`;A7KwF8b%|LXUQU zxpD)+-9MJ~H0~dio~S=Y)6VHYTlVtLO(GGkOn%8 zW_^44k~}46fRYDGWJP&_8Mbt^#99wJZDk2B+_9=PN>gG&NaRtb{p0bFdw2lJyCDz} zdGKy%*Znx<*lq~P@Z*S@umZnTY{V$D$D6)=Fy7=tBI)JW-TS!aY#O!US@wp}=#umq z9Gk1aF_t)k18*H$*$MpYkq)v);%AC|!$tEEi=d>Kmi3alegzL<$%v1&Q5#UN$&Fs) z2O5t{1%B+pNFgLs(aF^ZsHn&XlMM9LZ<)(D^ycSyh~DoIjhnoSVVpdBb@B4R2*8z_ zIxX~Awl||k z+FslB>DO&L97n%Y+atDF#j;Fp%d1MR1_7t`x`(zBc*ta~*&}za5Qsv{$f!95%ti-t zfl)c^e!M|uwAe^h2ghS?O!|+tB}56;Z-wtd*?4$?Ee-B9U{X+=s)dzARrEa;p@jR4 z_aZ=3bLJM*)2CZYfH`hUnp5Z$6M5LzP||ucGyXCJ{R2lOiua`KHG;}dhwo44=7UAIa7fWL|9lDjehfc>y6>;zlx76SU{Z@9<}#%SWm zIPhQ;U|MG*ZarGD=j4MRZg|MNU2{S2pUp^1S#$4D+=&&bgpmG0ImsFe?85E6AVBe3 z0SijEb*}ZXeF}DDhAoIBDYHN59m70@gSe|_We1Z@iV%C@Vj;jTuYB3HEfZGxmr2hlA<6gG$Y*K!z6*mOBrhAQ;fXSfw%^die-BSwKsI{}6L2Zoqs_pX5!8;h#9z6`1r=E4F5(nvoBN5SCx#~N>{ z09X#K4BXg8-nj|^m~1|3xCx5|IahR9UH{%QhVvj~E%5_`)%G`^EIj;>uxOBMEM)~uK;RNJYPkQA8UN?Em zD)OTXv-%M><&U>*jpg@jTNbot12DIEO@vY~%i)PBIsD`|HY#g)T;tN>mgRh?w$$+0 z;@y5C#~nzwyOLqz0xnj(Eo37$Rz#ZJplIw>{Cn?3>2gI*PHe3{Mn&jIw0qYWQ%}9p zf7l++yCJ`>CzffHB zEE}$6O$!fV1$;Ml!2~A2YS+dz$rZ8t+XcpgYmEQI5YGvGfQ1VA_&|}(07^?!-NLlG zMe^PQAVO{@g}EmrG~v#0jk~jEavxy^58oxL?-;}5<=T!@XtTJ(8dNcxL7i4O1&3E#(Rdv4BzbXV@m69Y)cCC`A!7k2_wHPCt9| z{f|FalMX9zo2(Z>ZvF^`=$^q!{QAO_=KjNid?hPWZ0r^>TGg1H1tJBxSfh;=2|oKb zIAe!_62E4GRXpqUZp^yfK}kGt8M;xQyR&wUjhOHKbm-wbASk4|IluReV1P&tNfD4L zM@2rTg%pfjefJ_9?Eo;I5E$P^#T!?*gt@jwwKd-zT3KOcz_v>U&K^Gj1OnV0O5%4f zW`r-;!u|!fhK&GF4GMa_MmSzqwH25dG~bu{)2ROVC*5WJ+`rHNWWwd)uOE#G?8n}> zc(4VCLK|G2{U0snB@UnZ(d2%C0t+6Cc(^OE*3XIoC{JxC%IUXSS{INVV{M5-)<`%& z6Jg5qIGsUo+()`C3pOLQl%$YieGXWo7lN>lT*1D@F%8t2Hg->Fg+(#bP3h+mVSj!r zWc^#aoFfgj57v^Dkwo8M)z9Xf9b6GP_~9DX)oT z#_SuezIzc~`$jcF12rUEXvJY}lb7@Was>x$?Xl<38g*kfbIW=|F$xH)j+nYg!fY`I zz8V&qZLc*0bbr7^s|9%@t1T@GgMn+r_ORh=$%tSXQ;Jr*85Dr$t~irH^Tt9*R6q%J z06{Q?pu)m9CCf_QFhR1SQMHG}+h$}|jOKI9#{qkq$#4saL6$ddX$~vqZ*93%TPEbY z7lf{A1{GuVkT2P&TaTd86^p=p5P+|z<<}XS)r{p|@FEa5k>Y_P&hQk@WbPK-ZF5b> z>eazsjc`k1Yvh(_LGp&FTOo*p%iBZ&A%uH`EW`&G%c*W!YiRgS1Uxe-;Vr97sjMHY zXHSfXJz*R{<1qfQwsZoFffjsJ%|!iOd$xz0hzAxEHLsj&g8<+(3~wWF33mrNLJ{L7 zSluQ|nsZ3AHX~^m#mRX)qElnCP6xYaCQeCHvp}Di;cllba(Hp3xyfsp$8YUR!``E9 zk~O9QE6^(OhA34+wytupIa>@b+|b*@Wz5nIrIjRqgAJO)>*mkKbsu{rfEnPG2x;Ko z90E45m*zky3QnDy_Q}>U>Pxi73YyS0#x;X^PJY`FQH+=>XsI?@@IV~HGiAP5m`PaN z{1IlTdwa+gX|LbzNknIZd*eI>!2>TRk3+#`uv6JOhHW7_7$_6fk}~ld-`)?d3PfH7 zbqg4*Qyv7nOQ2|_E6i6FgFe~r8xx3Pn@rhTP2oc9a1dX53&qpy5@>RxMe5_%TW+qc zZSnS!hbj*GS4-y|TVe2B=c_&H2{hgN{Z_(J+9ACVfs#Q3eea~#S{xYN-y?aLN{r;7H5D>84@jw!8|LO*KUR`1Ur7=ljJfS_glQ8((3vgSR zzoDDkoJh61bNFbtSAI2)gYt8@Z0mfSfzuaD8YSoBhQnAWgg8uy9MKofHG1oseifZNu?~C!sd88ER=c!4V__-jRX5i zKimO2M2QJdKbhLDtf4SGUjEkHJK<%k)zrn#aqFf}r!W8ne(t~dj@Laf7 zaN)6J8#l@h-a|DR0OK1|o8>NDhMlpR&-Tq8=4T-oyF zv!2kMOjR>$J;ffMo#`EW$BIJ#dp~Zg7DN5Unx0UPz#8%rP$BU;%PU!y#uUFj@AunQ z(g%n$i|Lv;VWy%u;K&hHRkFgPvER%z0oc>3!c&iN4(UTWJk|DBFu>_Kb@d=pw!+z8 zSWVB#w+aW=oeR#>I-+eSkoNU)TZ%X$z>Pa>ScF{>iT_c;<*#ozzmuLubES zz+3<>*47IC&c5~QiUvdK{i`hz|FX-Xb$UQHi|Y8) zC;8Qei$uHf$QbHZZ>Z3c`7M`C$_SVk*rrAa0h8>P%#`Ctav3dY#_OG_kCCdNgA`KJ zx>2q82(kw_;UCaR>t_6+rNmH0+Ws{Koqg}waA`dG!>{&n07|26hPQib08?1r(5y{R-u>iP~>@J*G@lk5DZ zMJNzh)YIzzCX($|2mV#DlnFPh(aK=`tFwAtAto%W1;B&4X${;Ig#K26w2t05Nbazah9#FaH%d2F!8SR%zhNtFJT97IPZRw_#*N%|GdLY2$X zX+|oXB~&u&U(*&CUXe;?{VTol*#x3wuS!~^5?cQ{;QUUjl-9peE2mBBe6PydNF}vN zBqLSVCRJAJUk43#yz*NAO0L8Mw zB3x?Kw>_#;c)sc1A?tEVFtxqX&1rars>to%8!1)Rza}bfQ~MjOLf60Uqw7tc!Kqd2 zl6AXIZM#SnyF{XqT6cq0?Gj0+)xL98?vj<1r0YX{1%d>+k_8Unbgr~o(q?6?Nw*ogW|0!rb?L_XgSz@ zBQb0M@y&*9)ue2yQmV8zQ?a*uz*u8)jcjTiW_PCT1DFO#(6ect7hO0R$ZU@Y4-ggmIl|id^yfsh>(LvM(R4b~^3p77i(p_u7<+h0SC!nddc#Iz96UfJW4C5hai(f= zM|T#Sj=DmuVC1z(&?6+fVgGG<2gV9oiai4~Eu>g0o{tfAup17Eq_>m~uHr*W{vF~z zrNRkbTwR(c9B%q`Y9`9CHHEhmtB0P86^00hmWe}SVj{yqjRFHlg`@S(O!WAXmF9TzfNa8t^C33GN+VSjzVEUS7f!O`q%<08Lin`R{GB~mC4e~B}f|QZb~AQU(+3$ot=oH zFeV-_SmCs8b!uha05IKg(i7JL&fSA7gLo=Lcb?EVXxw3q;Qwq`TLQ&}x5qks>8Vg% zq%C*G&j>B*jcnE%OfF6hPTo?JvPP&q55Y<%-AchhS*#NV5x< z6@tajc(DVaO5`TG3VRlf!sj{Lma+!r36`uMVuV5^#{!2hBEq)4++jX-b#W-PvEr;! zzkwM|S!j_BpRgLk&}Dr<_g2WZ2w2&MRy3VREEs3${V?E_uK zWR3_7))2^5Y$!qySfc<}(VY;4zZwCzO6hzI`l=Ul5K}VZuvdGaRxxoR0(mteA;sja z0Nm9Gm{p9gj6hwb0a-IW-D}dufvWalsFIj#45Vrb8b&%mmBSC1q$52XIMu;XK}S97sAnDZtkkw9!aY3> z)YAl*r^!g|=@3upi9Jd1PF>)p1ga5ervcEW1fn5Wr$NA`1ezm|PLB#g_+Q$tkO;{1 zNMKBl1jIBEgejF@Bm*$D@JcCqW*YcX7g{MvYWszOm(r!6RKTS}lMa)?Wu^nmOa_(t ztN~^24JPv#ph}-DpwcAVyHp^h;~M{Ut?`EfOPU0fWFe4J1e+v~BoF>5fo2>aX&m|} zf&4Hy(m3o<0?E&oHKdkCOaenX00=2=4xSEzlz#9}2S1t$eB_29VUff^k5aSPlVL5U zz!-V(LF9@uG+M9FYO6EV9;42sCZ;7DJV<1j@1cO}ytfgl=&6-uT$1V1zg zDU?j{UeH4mVTTTY9GV8dGX-iW6=rA>#LxkdD=B@O1S@o?<}+z_ra}oBkf4NSB*F;o z0|cdXF$y8n2M0}LrjL-a%k6ajpu!1F|V&qv~xMxl9P+aG{? zror+=r@iLI=L^a6cyK%i2i;6MiQ*7E2f**d)vQU-I~G_c#>VagyEAFfCZ5-6P&)~{ z<=!wm50>Q#8#q1Bv#eOA&#>pcsBgivXrTH3=axoEv$~;$A&YsE=`Q?sS`fr%Lm!k+ zeeT(mT1bHDwm=%l?_)D_h}zqk&F?Wz$vbc>mN9qA8#%_;{PboRd z3LQ&zdrUiA9)&q@`bQK0f$_g-Buz85R8+9qLtxu8ntrra-6X@K z*4W#$-EYObtk{ZwMEn%>oXo8vK<0S5nHZLM|63-qAipIq8dXa#hzKp5jbewn6kRJ} z+?-7H;e>spn%*Ll+b7%IJcHLdkKc!r2;Ga4RyM(CDQ}&jWR3_RESZ=ESD>@&K!Crtb!MA3%-Me zd?~IfxvrRs@-={O1L+&28r%myXHzw;031s3pUhE{rbu%Ka566oD%fJ6onKL2=))RX zkzhpd+o-kY6+EwBwcl{fOJm!M)U4WaK?}?@&`o?JbB2vGn#l9s7{&F`UPGXvTyV9) zRpiFeo#N4`wN+CAt94KN5-<^r8+LiYS7z4^LbnYDIM?jtUI2a z$(}Tvnre5tZOaTMRBsw1G~ZWLD1dr_+qE_;OyW|%wU~Zp*j8*W@fBeoONtUV1%XVT zCtEF#x#+^VFPUaE*8e5*vcCrRetDZM@YGw3oQ^+OL+I(>BX9d((x7d+J+z5Fmoa_SHudIen+Smc(%fqnUjJRusLSvr= zY!Hfx&}$L*rg$8L%)4p>i~)FUz%#2#Ep9A}XtcHTjsnM7I#S)N*PW~HUc~HQ&{+$f z1~z0P;NOD4>P3PoB{NRr(Qq(VcJjgGZ5;bWxuISbu`^+T-6?MU3r4D2d)6W>P2SgW zwXUdW3K|kG_jdUs=%qoaA?Ok}@=@hHs+nWs-4r0 zlo&m{9EyrrS=NaH=pKc{U!job@!d{-uHkE` z=YrW-jmTM5>>&0AC8h^iU{c`Pnc}%@AF@8>tf?Se>4p78x4QLvrCEnvoR{C%mWXHF z1P4u$(KScc7Q}ai zV3coTH4~?#dlXdFSCw3{_dEr^qSw{V(++k+f4&18*JV=X88H!cW`t70epj;E()u~k zC(K;GFyWa=5zD$~Xd~85I(TrpBswFMTE*Lqi&i)dF)c4@F;@$aXimgnWjH`Ti}Bks zrRaM4pqc0FNNt)e74wSG9O@~UUz5g-iyT1q#k#kX^t^aunKE3Hx?VA=LUn*fdXA4_ zL_DJ8W5v6+fJ9QaslLCeXJkvW4HxDdLGyqfLK6`QV%IuB=rscX;xsf65bEukJXL$c znOcpT*G3yvAUAxa05g`20Yqww+bepe=LAH5E5 zTbq$Sdcvd^iL@sSL28E1JT2w}1=ggfl;pIBSbUO^lm>$`Cx-ULNcDuXNI(4qh~aQ# z8&L)(ifH~ZLC;eoz#4)k0y&urrQ!>(mo)Df_uDmD=pLA=&@o;k!Ol13WK1|iHVXX>o%R0`8Cn@pwh@lt6OsYUbL1jf`bIhND>$3B+ANNZcq; z9-)Cy+8>P009>y6Ii97=PnAz$zoVlD zyAJ91NDQUR_l_P28+A>yjY&> zM^U=XPam5Cml^&i`|00@CN(CT;9JKQUAMX^+}b(BnCTu;(E|c)UXWKo_<2;_vGO4; ziVhKFBuj5%6CuZG#mF0Zt*53b*o<63LR0$n=aCD3wDG3 z>ww*~OVq;HjBhk)#&cxcZrH9(;4~0BSX5z@q+2`|r3CRs{-?>^lG`O0*f?Wlk?mVY ziCRPgkwfF4S%(gC40oq9(r*)Sc4B#L2H|W7QOASFdG*hR77OyyLunW0Y2rmR_Zg+AMH|kuHALe1&Xts@+qLvA)WCdx4XSpPy;@Rw$XZ87l{6DOcu9kn^N^u zk*DMg9vg4xcFS@UInQmfD<-y%EfDJm;G+K5-v<{_$R!L-PFgwN*|H7oh@o|{w-n4# z8swP+&`!7$J=i3#`f`kAIvmCS)a#FuimhRf;qEzbhb72JLk7fJma>rR-E~R58&Em(uz~2*v4vn2F>37=Ve7 z1C=-(BryiqGZ`K+Eu`)tfDsc`Si&vSVG!Ht`5W7%>F6h~JI`3*r8XjDdt!ry0}tz< zq7HH=fS98SjAFg!POh@INnF6furT9cYo3*`3n_1!(#|PFw&^!=k;$!*xv{;V+J)fp zZrE;)iXYzJPAkTachrKm<3Z6I%e=`|Tf~0KP>hnbVg7zEaonH`s1dGKvRd*y2W zZQIyw+qP}nwr$(CwcED2+cr*rb?a802RK!Ef&5rWRx-z!Rfuu39~4VioK5(tGjuMz znQ91}8^~~(uL5iP%AWp8>dRvn*sf{AMY4K_(>+vpoLXuyqhM|5)@|~&8;J>`^;)H6 zJg0x%ah(n=UKQ%L>_Q4z)u?O1U;TQL?Tl@{!1!XJw;?e9tw zf>E3wnwF}pCjZuMbsA`a?&LaM2+~;?Kxakg$=tkLyU?=xwW0;f0t+ivq26mquknZdXtfvobPNY5L73-5wW2rP*dS_lT3A)Sm z*yk1$0}x-%X$RD~Vhm7-7&R3}dfy#Iktn7Po25(X~oMGp}5H9Cdl31 zpr7p0Pmte7#8W$peEL1{9U*622(alu$DA~2q$QXt6M!#8Z&o{SnBJxIJ2GK1OkU9W z-SJ&DrDsYPGoa+=2%yhQ;Hs;(J%ZJ=p~oK=VtYLwf3CZ02vBw!+jay4J(ZT3&Ca0z z+G6N@=Up9=Yp*=ZR_83KTyOt6`08s6gsg4P>m(DZcbH7u>3p>}8e&f3b`xD2*$M^R znjGRpuR=RQ_O~^MR0S7}c@%CWj?L=Z@UhxPI6zLKTwp=--dX?X`8w2J`CYi$7pk9M zzi>}G`%7trGHvv2F~#1nL+`WfdZGra}ENjOgUe z4F_ukdQ;sB5G_kvKVikpy*!gc;5$xE6y$gduXgfyR%!qC z&Oe7VHaDHZVd|v~w`h*j!_cm|XKx1~DI!;48R9V!FX3cI2%vVx^IP`YuNiVIIs(`t z*AqPsrGGC(S~9oN%=8jDJv?jGI{IxulFp*Y$f!7rLszkDP|h$F{GH*Cyat8T8Xi#E z?}QxX8l0S%@|Q|fw)(;pUl`09e-R<|n)bb?>t9ojRFpXoOT)F(4rWh++fnR({m=sT z)eq`x_LnI4ykC0@=xH+7dGwNHTz0y6t$yZiYI>|R^>g;rgkDMfvQx4r=pJ=fk##es zz+`gV`8HQNDu`*2%fVa0QjSd#g|ycp{TG8TSs?uoZu7STuo;XPuq8W(#1GUE8bbLK zl=|cIVz+)rL4*(i(}A-}=~zCaX<>@YY+xk1@@L zVHiXI`f?XjRQ6=NZp>cYs#|PXrl-uts<99lDQ%bRn=F(62-A&2DOiP26fJK>8Kmaf zkf~j+*+*KurchUe9+JpBZA-K7V}!E;7dU&U2}?nJB5zzR1Ll}Z&wSGp#0SE@Vpa`3 zh-PJ9@sheC(RS7^TB$il%Y~_iVW9l=)1&h&p@5TyG2f<2orq7@2r#WwYEAZn)N(ayL zYBjR7T6BT(k%CxV4#~Hv$i8Az5d+1CYE8IvGg_^$Ea2l>6I*XyFo^fzXy8RMejpI6 zY%{{GE7m^HrGF^E_*g>h)f5sV_hj;UfLM@aUmSIO_(p@D}j5(XR$w zD$;l4db}RbN9(zPe3DCQpLKhED^zwrJ=(Bq5Y8H9C3T zZ~rZ`GtPN^9IHJqG!J@Hc{xb{{`z5s|1gzsH$O`lIBeV+|H@RiWDF}bVkTELOIEJlwWyIJ>nV+6ttIgu!6&VtAfWBz42P%w_nVq z-$j~gSL^Zfe2Hhjdj04cO5I{hYe`>{-LNZ*A4iM`RfYP{Xp8`sFHii{&skr{5qV4gZ82YlEYGL$@ z#Z5`ywa)D|AeP`rC?OMNcI^uLz^3n7LAkFaPxDA@tL{9$yq?IZengmMeaSwQRFVT= zeVMQm%pbbh?e+f1);{Vm0E{uLD(#;xMGH>-Y5${APir@?uxxMZTr9%E({3JxEp+z9e9951Ea0j`?u+_UHwsJdxa2H#4hcDwEVpS zNfW1~ZI~dy0Il$X%9$}w<)n({$Zl)_(X*gTWPL!%vFp)R2(9V@j*0f0E53ZZBJ19p*-K?osm@T7 z%%!$(AsZ&-%51GBSStRY;4VSL1h!izV$|%h9r+A<3Sy51+=3MAV8}Zt3@YiBlM7v; zUH7?7(P0P5A}Q54t1<0(vnVq;c)XL}Utc-#{ojn5X(FRx2td~;ylu!z)J`PPwD#-9 zQRTm|6Vr}oYBzo*;MR_&x)H4TRvdX*Y@-SpF7!6Iw6 zQf(k|V=7%UD_7PEeOBb;M=&?sD#eM7ynOEO((J!v@}eR(;Jl{O2UE%KLZr=w7Qp*6 z63j}`dkk6=sUsfLGn(s!7HH{N629vDURm;-WW^24m01<80*tjwRQIZNxxkz$RpLEMw5UgjWv#s*Xnvs zCn%xo+-f=9$wM~XN_U)p$4Rz2$Msa1UwN{w2bHmX)=Qq58^wIZGZT+;?`z4nYQ`&EMnY=VS#Hso-1-kagP%EJ~wT4g1P`J+P})H4swx&R!PP} zavCFzVk%RB6vZk^N~9)V&RBd55Xf@1ksxhOQ|_MTWVO!tW8%yGD>N6rk1xf2!*+Wmb&8@ZxF>g5#du? zQIM#eQAiVTU90FCu}iv#{?DM;+(30*0H|jN5z-=qDVY&S&x3YHRv<|l*N5wLK#vH7 z1jI=hIcJwoq2s$Xm`r~PlC1L8a}t%~{iU%$Z4C*_!^hP%^zXs^FDnR6hMe4ZMs>aj zCTg+%(jFa_Xwwi&#RI{KzGOP41QaVV+|$n4)ei9}PuY!sI_CZFs}WQfMepLGVuw*@A5_*3rFb21M3+9l4hq}NMBDZj+P z?)S+Si8C+B6c#KiAj2_Cg0`9F99CU`lhP_~tPBl1R455esIn@i8kxxo*)?uS`Ruh(sL zD1NPm%Oo?J0!B3S^aFSp$B?HQJsmgN zQhN0cLOmTW$jt}z&`XFp>yYPVB@w7zLqOvV5^rHjciJjv?O~;l;M{FDSGsYl8!bsD z4HTLvP0%AK@WBKe`EBXu63yRuU^07ADbK(!ZhO5vGPzoZw)x49?ZIAm+^t|eb1^rO z4nUR)VV43lpaO>DVcfr^Z1;%J+u|T}q!n`FT_2=@UYhKp?p0A9L0H%-Ev&GozKUwZ zyu0vtbecw_IV4e5g_uE9SqnH4#tSw++eA^8%|^Ix;8(qA)>-7U{l;6$GdNgHI-C!f zx+|&bX`{N`GJFmLwl7hn-qNq;M;C<6b(Y_7qh8;6J2%&BR8^3`s>d~Jm%g75BAVqH z)FG!1?x@ol_fjknN4L4Cn-=nsO-qVY1 zdG#TtY+}+Ad4hD1K$U)XOLSHi%A~+ zz!K4U zNy_1E$7^SNmf*`L)zfdG!y|5AI_VBBO-Qp>TXI~D{cKx7VM=+K#T9(~q!4wxF;j(B zRh=w^fi-z2Ra$$Mr3=ziB=&xwPM5bmY-p7ZTe^U~qs=BC!!}>fk^OtnR||&DTV|3) zDzT@F9J;*@u^2u`LrMe4`Q{9P078VKo66({3y*hN;(}+%L__OqI&n32KX^N!T9iad zsZa;g-F)>^nLEtTp|w1c?9hp`#`z&20B9DzVH z#R2KD3WR-Dx;h-hS3gYR^+0ArcDKFtC>Uj0%J4y#fmkEM3T#z11+lZ1Jp8PRKA~DO z0#eW=dtADs`v+?gRPEzA$mJNY*4C)WMM|ALEvsQ1li^edtoe9dLBY~sp_jvMe5?K* z^mGp;^9flAa&npacDh9%gY>p(`K9()C7QK`OOcE4Py%|qUZM5|C0cBU$jw6P^=EgD zxt1Jylu}ZM2BOO1-%F3LFCkSvPZ_C@7SUz|=w)vb|Fvcf;-%;%GX!zT!EW?iM&(@7Nskag@;x+G@o z9rSXVx}f5IkVqxjBlmP>U3hQ0LzlgO(Q4##G>{|~A|1#oU1h-W*s5fZW0 zS7ufPBsRdeQvKKA?%*SFAGzBonb6c1oVWywFk61gH^B{PCkd|{x=Cw=wv%d=71J|J zH>#{`W}tEQ1okhzAIhzL3($V(se=?ijHwTotnyO23-qeymn2Ga+@;*n`b)SJKsr@jP`&lo|;{f|Bs#DhfbB$bow0xgtQiqDlwSsBR1G0zZ z8#X5KgrEEWaO2oX;_ZvroM-!hp-O(x0(FR1UH6ZknEEr@Ws$0vZ;x)9Mc2p!+ynSx z@|y>MO;!Wu3+|11&TP1cxH}vKmt@5g0A6t`S5F`3v=^!PI@3z_&$u@h{Wb*#fQTwb*=WJo>;T^a%_%up30tqN>T`D(y(ER3_jBd$V>u;ngS0eV}nk~?G2)Mt4-5j5} z-b~epF{7=l3dAi?xNM^m-=DkNFQJsLa`{O_22?Bsm^5>XThrpTgHOdTM zHnP0%o?yOBY3vF`sX1oY>?tTJU^Ga3k7DYP{sRw_mr56qbJF77@CE#I5olC4_r*pU zcd)sa{E9)AE|Uj51|PnAn=lbiDje>|zSFy&boMQpfN!uPlXIA|G~504sLL7={3IzRBk|8j~Dg((KoL zQ{tTkOPj9EuCc5zm@Ckc7+13n;-ygs&TBi|k(-=`Q5>&zI_7a&Zb(|ZtN>~-B!QE6 zzm+v39&f4ELI0FHog#|>Uz-I3N7Z^PCiV27tFg0;n(pPOhaiGoD(B|dE;H3xNS+(x zFe)YRyat>Hqwe=ImEUcVX4WQrkHerGO53JK6INfOG96F9zoFtNl|n$OB|+YTwup4v z9hYK3a~_83BVZOTlbv|I1JdrftS0L~$&PBhdhD3}Ja-1F^70Js*X`Iy^d>v!eoi;K zOi35>0=^+Y(cw5Zd1mLSAm2-B7Rh-3m(Jqz_zw+T5o_c5XrzQ1g(Lx5`dG3NuN!cG zotS=o!l|tW>^FBW4psP2V+&~dcnSKdgM&+jx6O`n>bqzcmXQF_UF&~$M@;D4fX7Y#}CxfQDw2jQjG&$nz2qr@lnF){0Q?(tZ`Ru z;ue-n;d1yIfQLC4d^SGF|kNYkkIv(;0M3O!VyHr)E zens0D1VE=uf&CA&ZbW#E?W4brr&sjn_u6G1S1TQK<)IO$t~a{+H1JIo=c4ivvu3)D z+r!mGgE*j{qE>;r8|-z{5Ia+#<^m>ZJK&nryqhqx>|y*xx-GpCn2 zfyCJ8F0khAF$}M~dq8ZwP=vK#4_u~)qjTR+5N5l_LtGOdh&j05=gt;KiU$D@(Qq%~ zsEb#6>y4eUuU4bA?Yw)h*ho4X+sN*k1}w}nVUW3`MpI6Q-^ z5T3+sbz6)s6b)jifx%dmW)E;A`Gy7sA#>V#U)%wvQj{9Da6=o$I0uGFPj^XyBQ2!> z4g*vnoYy}=DTT%g6X0atmVq6x*P+4u2ZccPCfXp>!M5ViLtDj;brx`5NA$YD7)_?_ z%lT5o22QY(jSN5$FYaG5ubHK1CV0G6u23Eg4+n&hzLBhpEM=KwBBp&Mc$|}>-TlF! z2@DRLS-Qyaq+!j8`h)s82C@06iMdH`Obd&Vfa?TQz`0l}6c2hrrW1~ds{}FZQ_fT5 z7&aNK3pxwYJFF2u8ptNlPV?kev0U`90%R<8=~Y(xZ0$K`ntP+Ra&PSh8@HEUr7FnT z5ic1A(W;&eso3q8`k0~>idNNUYB2VQDa|j59X?}Vc?X3;-0f{laSx|1`J0((Sx3;) z*9!VOgW!c*a8~(+UFNOEBKT!mnb`kaP7+M>rI=s0E}JmY#D#?mtRiw-)4f$zVd6St zT%ld4eqP;ZtJ)l}d6_7Ha{sc9!a~1ie>RftsT-_WJ*LMyohyHK=pd+ZRerfg4`n*@ zLL_SRrCK6~9b`wy-m@pVgOa(VF3{2|du?A7m1X6={x9U^7T#@Bj4MJ%i(v>ZGg5+KexE&2 z=&`&(-}8DEc*;LO!p;6 z>G6yd1@$c}*rhWbQ`K3^V`RU1x40&aW1|=*GO@*$6$+su|j<_vj^%_fX!I4=1W`YHibdXaMoqE3Xx*WvbO+jon$zT-b ztm>N_cGYWF#9?!#d?MM#CPC~+HWEJvC!37Nig=)PJYH<__5#uxwGII$-FN%9qXu?s z`P7LBG3}!hUVke@7|hBUCNz)a>U9QTLEBmYE*`7OCR8QiiSjF12z1HCLi760E3kM1 z2TeUiity2fTY`@rP5#M1P2}Z9G|bRHjy)2P3!924{gnlCl4EIS66V#=4m8U)B=u7^ zw`e+?M;h|UHmn2eOmu|~0!?A$n!j)|j{G!Y!xcJZE zlP0q7>WCNE8x7)@Bx{?uQ85 zlRyitxW8{8J6$)v8uJN6M+Qxn8uqt1YWAp2WRIFs-HKW6ukxnsZsQ!XhK9bT zr$?!rQ`z=&R|NIu8=*$0(n`AJCv-g*eSJj{WLqJWi&$G>d|nr+a=}t<(yVVK0=G^W z@i+M4uIf)6I(2DBJd;PnRrbKzJ`@%5+L7$Z=nZ&;Z7Nu02>i<}hsa1h@wo+?$GV%4 z^(}ocGT0cJs+S3qe{Aurv4mZeqFuT8nW)wec=QZJxxU-$Eej@T z3WwZZx5?7$Cjy!pUY?2qNkgWR9i;$&R)v`Jn%zPX3drRVJ=+RrJlKTUL9%inC$F*Qm z0OVWn-38D@+cFon5_sznp#WuHWlRq^#ReT=juR**)x%>NbzFVQ#ch-D#F7FWSSmI@ zH+jvgo)kf=#^YP6+Jq-S>!^xb!^n|^9CS?UH0*R@$PlPV1F~-~gA-#5wNy#GyQGPC zaS%$2Jf=P@Vn|kF&}<|c7;RgY>U4KSuNENs`*^7anmBI<`EC>Nk#8|uH zAet@PSgWoJ{%E6t`YksPb3ttm`{bbc-e^X8AZ~I1sD48bw4&mCeAD znj7=B4=Xvm%O6#Cie=gD=PA&oGioj=*E(fTctvY zGX`*}i5EdU9AB*URi+`tAnsntM;A#=y7XE;-m2ir;I)_9vDFVIYmpl(sQ1gIg*g)N z;hzqc2V-h5j%k8hJvt4Q{S8{SuWagb<__cbh1(Kl?XGW(ndD0j>;kC?*de1TmgP&B zJA|~5Aeo5X;1#A*DQp3EY!0E>`t;x$a6~kNSfY#47kgRdPrISp#;Q?_llsBvlGBQ|knfCF> zKVz?i#s#5jBe!pmN`EAbTLcpJAgh6`cGGk+uHcCUqoM^BMZ1CjX%%$=jI&l#37O{hDd$r!yV3_gY9l2kG) zELslxQpQkvvR)*9g~?nZDHu&IL@ZYkBc&~yO@MAAM!Wngy|~m5K=zi~FW5^wlVJLm z@;M=skh!=!&0PbI6Hgl-G#wDvCZ>8A#gkeK(_4WiwMWPkq#lL{;l@YG<23S&;hY0! zgp~A27mbNl*M&#hxocejb_(!;5#HoJw*_h;sio;CMg{Va^R?u{SKzbeJyTAw9Q!E&4>Iv4)1***E-y5 z8xjvQwaea#DhYgd-reI_L}j6KlLZOF>8?rf`e89?a&p|iRyg-=9!S|I)+GKZ8iZWt zg6aL*(Bo#M{O3e%2bu}+X24vNp1s0~n>CvLboB9TYO^ynX`^`tig$vi$lo0gS_U+! zg^Dd`^Pdsj;OntwJ-j=@0iDK=`$uC)Gi`1MD7Ff=w!dVOLss5L)++k65x%-QD{Qk{P5$`HtsxVAlBEh*|reVJhJvYyU#>B8V4?X30mTBHW@FmX+ z+StXS5NjTXaKlOrqIo;D#!+NFd}m=0{_K&b4Ty%EokTf?sg6SBP(g6OCpira!SI)O zsN>ksp035L;KeBi>>_D1@suxi8on$UawM>;I!$$Ds{hfIlmifRE4fq)v`&6NB7Fn~ zoVN8#ZOZh;rBe?u2JgF7^}Z}<@gZ>lv$*i+j~Y+;5l80!%tiv-Ac%18TY7fq^Nc&X zHZNTs)%ssDe2u5^^~H|q*GwLVp3HUQV$uc;ar8o5fr!mhW*_%`FeIZwsGFg2JUkYL7AdxZCJ8zJQ;I((=d&!kT6(3jiS9!Xf!CjBWfAw! z{^=pVav@Y0$I31q>LOGT>hM*4{Vyk;&=x8mTQrHBdAVCsl1&i>8W55<*$!oObg>tN#(9|8QEY$=$KVeBFh z_9<%qm6J4CmGz=lM}J*Q$49QWL4Pano}0wAo*o5`f|qx11}NgaI};CyOofFb!jG39 zs104q^Y^L#G7VL6?G{mJS+nYfGx&qSfY0!I>-{f)f~YOR(z2URSplJBmLo#>on+ux zzDr2N+}qM;tlC0(fx5U|8DXPfJvTHfUtZo6A^D-P?*T}|AgB}^Qj43?Vg^EAMbF(y zu=iD>Rv0N;FKXStAsTFm$z1yAJG6SO<*525HtUC6z75S-ug$`&Yhii^dyX8e*YB#Y zkAO190(7*4$!Ef(3)aH)ZeMkw##D@HoMhv|@6xg5$oKN2ZAzd?pJ}gW zwum%X{NC%k1n&?tNt^JHnvCFk3|xo71W?mm2T^Gn*wAU!)!K~vS$!`rb*xEX%#3x_ z-2k+se~x%j|0yK~)(VVYw+16CS@grm;H1}0yh;D!EoMK9G|S|!DGrkeX^5$#z)>0Y z`gIgzLy3M5Uo?n>WqrU0Jk5D^r+M=P2yt0;r9lPZa>fNnI(p|KwP>JgbF&bLN?Ama zFd2UDJ==|(+4~B-xUN}&Urv9yB&J}({5{G3T5o{FzC@&Ypx5m`6D4_;Nk8t1+L(8LKa9W@v}Ewu7+}35!(=k<7{)r?LtJ@zMb9d5jGn<^ z^X}do#i;L8-hxBfAZL2am@edc&lmKPgkU>B<|)vrLq?+Nlvon zGB6n3&v&BU^ryS^Y>}-Ug#^NZIQh~>t1_v!ci{ebi-LGH`W#&Q+=E-Gf;7h+#N~+% zrLoJ0z}+oxL^4r=Opfw99i|?Ixwnq_A>c|;!aeO;YVJMua0O@7@qFF&SoNIz*`OKX zvZJq97H>vJbv5oR6wGk3`jC>XPI?H=KCi}kyusxLFD)0LsMY7YvFGf)=VTDl`BRUK z;Of#XGRb?hN`CnR`x)asf*O1hy+1!Rq^+T@9=~N9y)ZH9SLUJX6YF-nAgQ=)KG+^) zm_O*%Aoo<-tS;tOXXhVD)c12m9e?z_TIcs+X@AFOyCcy^`me|n{RPcz!RQwp9hT1; z6xV3O(6w@};5n*)6U7kR*}}SW+;FnKl;dVwDQP1#irjJEmx{mcnLc7z1pe`RH7C7m z^ACenurCB zI)N?H0;5UvG%2~Kbcnkvb6tJI(qTiqXhz0$%OSYg*=pr>(ZcK4MoX?Ov(;OcpmCe& zPq@qx`1+TcXh{azzhC*rXyXJ7OqMr}Ya$xk#CXCf=_VWl{VSJU6`4M3-%~6x*3aF0 zkTqpI1<#N(S?e?Sl&(nMu-evnk#Jkl1>4QQ9aCo0drl#WJhbnr+dvYmjZ_@^0gFwU z)^`GpdAB6);K1sHn`?Hxxfxo_r8sa=p^B zZ)K2r&&ctx<0KP1+Qzf_ZObC%Y_I6~)H`(MeaJm@lQEHPK-z+W%w+YhYt40Kw->Q` z2)dbx8*n$dlLEpq^eme^$eFT1SHd%MHcMIa9Z&Y<6K9!0PSThWwwt;^(e)KzVA&b84%ik{X^)AI+LJ@#Pk@PFo?3fE9`dMD%xOL{G|rm&G83#sy4%X zL__+-sUI{uGMonu5o6JR8UB3KR;GLxqIn=HaI~^+px3og=v&vxOxxS9C_IHtWBKrf z{cZ0oc<}+Y==y4as9bTjok2zoUeTP%hv|WC%}wfq1ae84&A)b#CUl|}uz4X@|Ef=Q zl&;zAi@@{jeK~p65St$?upDt!nPcT*-daKBuvLRr{?x69x?*(5S7;7e2UB`QPcE3o zHInTidi=(l6Tvf5mB0{HxcxD=#HhvS=|^Ir!rH=^OI=w&+iE$ibLN5*?P1ea1p73g`&5;f@u=;4jjh})g{y< zHr9iU`XIragL_`L0qQjlLf9>~F#Na!j$JzPr9 z#fFt#ZKhyjejcbd`yMx>gI;AQSw0|S@CnW8tWF>8lK|VJ_F~{bJ|h!CmFef;0z)z*K{G=d0XM~5M!+(o z5+O*B9OFbyZ~`JbO_1k=51#A*=h$Jfo+&DD!h_6vz<|j4(09IW&{wkm6BoARK~p_r zl<$IvSoc5?o+CDJxdq(fL}Yu4=jDUtdW!Nmqx?A|0g;=;z+E;q0D?2?lfX@0dp|$N z9T$4)VJtjrr0+Ree|CDV{m#DV}zs*c%i=$OHbHTPxQiUn7m{6l=)AAR$ z(jtFURFSQ#uw-M5R_nkat#-@(9Vb((t8yK4qJ(39rCaGQP88p=Np~cE@NL$lvlpZ6 zvQ(4L$nSbu8kdajtiZv8FPJko3O+GL*Is#J)bOR+D;N~tZqaSc0uRfU+0$qo26$8!y9Dw6*8AC9C++=g{VDj$_L1H&G^B(blO_! z936UbMZG(kR3itFlNOY1+1n@|G_9oqi^zYS!`FflfXteuC_qd}tTlM**Q8ZzDxHi; zr_;zOU5^A%akiSEY=^KNu8!{zx}SLn!?dEAgSt*BL+`YJA`2| zqI>2cHj!VP^UvLr*Vbbea!Qe;W{HKpvBO~3SI>~c<<0zI-b<2+0_I|$C}jN_!u~8p zEA{{>@QISYJ$dpe!t>yIF(B$P^&~eCqfkJ#iQ!-a8mm(JWIZDGN#>!9F1q$gXVLJ5 z%?qD~GQz9kOR7lg1rT+zC~8RC0pWER4EC#rLjA3wsHjLQHg}#Ft**}I@@~zR7N!z- z`}z|6*tEbWoaA;_A!+R%BIs5OjtJ)>kg9vPob_M?bY!Alr0PNAi*Bw|E&2EzuruBgU&q!Y19Mm3oq z26g$UQbW^;e^sqzl8$K>chXZj8V*d5C!|@bRwBnuJa5I)8=epDn#Rp1OJ(5V)jJC= zSzFiz04+k6k(^G3bCr)R$B_=JsV!LG<2PDGn5mX&BaYHZ!B(gu4%$h{nrlfC@al(!PbX~_ zqyOefwTi=P8H%GFtJ-eNP9hoUvU@-HBpqv;efq z%APzVB`@EtfvVb6Jvlym4GNPvr2#h|sTh*S#P6&D2u7jROW0jV@G)}qCIiZLYU+qS zXZ3)dh)O@J{18`KyJUrtG~7ETG3ws3wyo=j<}jt4nQ@2!JyciE(loqf)|j+#K7dT) zBh$B*vZrDRV<%MCQjs|~yp1Pz zFId+5D(Pj>XrF#2R6R~XP_gNefmqSyQW=SSw+T!L<2 zVAhzqpXdvR7%y?O<$U8-n^n~pU?!o206_s6y6smS#3l2EEpoiO+$UG~Rnn#9OGB3o za-jZ9IzJjsf5kw*$#m%6YTPQe<@ z7e#`FT>?gTeOB{>Q^K~BTeS55hN3yISOw4jA0y4D6fM64Sh%8KVFkGgjgup4b{VAd z%Pmq#$u!91@>lds`YskTfo5n8sqeE!=C*~vrg^$meuw;=?!^k? zhQc}dg-_I$eoVFz?J7$rKEkBys6`fVaMONay??0gJK175f>c68mDxCTwY#&Eg z!~ueH96>F*B)K9YOZ5zY5f^Ln%JJQm&t6kbm~O}y+#I`*sCAiPbW8bMaD;08td%N} zAjha}GjA?TW*;V!rGDReUvVDSyvMnUjgJ^QHhG3ul(L?6!d7az^4n2WO){-24a&nO zrEa4&xP#tCd+tRiPYLi>RFa0RI5G4547ZtQ*j}weE;CM8Q8+}aubyz06nA{ z`otI}y|7NvI`=gX7_&XU{?<>xh1+9RC4!SUEzvT*$$iN)T|%VCU(qN9Cg|n9dYzp{ z7ktB}Y)9Wu@xi}pGO+Zy|DtczWOm-IdGx=odI*vt2sFuuAY5tfPxrfKq(?+ojv@?m zM54Ui#wKScQsi0rFXgx2rpmLys!8C$e+{b>xTzF;Kv+AybB)V}>a(KA=aRe$+KJ>4 zobD{0*zQj$O{E}F9o0v-%4Uu_s*cksU8*$Mn>!fU-ZD4Ht;F2dF>v;5cjQ_XG&6%a z&KtfBiE_-eW-W%yyci#7tb5?C0sQ@Egq!G^oq4JHM^Pri>NTt1&uA>NGm;Ns<)m;JLrU%flq9;9$Aj@ND!65mr8^*KfJtTad;e$rus7>(K zZoH|vRmd)nD~m$Aez_#8n&#P1o3=fV0pxiI!>)%Qexvt!gvYrYaIu7ZY{0eVHF`FX-D8Ao~Ms~SH`^zRFD zgFx>927x^vTPGI{TCR}m8_+KTC3d6E<=Fz6LDtCF6~0w!MkL?wDfclAb4fDffY@z59tF@5H$BGnf$DXKv{@*6}y%UIN}y%Catog3y9#-rN6x-`~d#73A2& z>)>HxPJeb+ROPFlY%aH}L4z{dlKs>?NZ1;r8B^$LSAy0c;Z9+zLlmouDo$mSs$v6A zUj6nrznPOuUID-m_XygJ%Ys%6gi*|L0-bc6Em$Fbpej{QHr2QvvQvd>-|VT0*;29D zQE>r=quzy{4D{>440M-UJ~E$CdFP3_*gLb$$MeXC>4rJx2hAQ&$Qci_!3Sx^WQuR+ zfR8TMuKe(k%sYPGc8+rd#gw}^7Jlx)H!`-pvY69yms64e z;t~L&vy)SbX%6NjZ|50DQ0&P>9?dR}AnOeQNjb&Qj4m#>U_5I$BJ<#2$pmqkriuBb zP3GvZbdwH$kYY?vGGv@I@AR5yT-~Tq@xsPqHYL@yG(7Og+A)dSr{gvR{Hok3YV_IZ zzP{}^QT@+O(6&C(-A>eJ-{~QIr-zIt{d8Wu{O9S#Q?1&X&KL*G1H?4Bh(X8z?(5Ir z#mzOzDh8Ruv``2N-Kl3|r+bj9l@^^$N3~|_as&tW*@^d)cj6tyA+V~~RLsl?1~*=C zs}9aU_Y#>Cdsgvk1uC?q+;8oqWN_npc&qtT!hd#vo0I zJTZDAt57wnY<$8k({_u%_S2G3fr&fX7ncoc&M&?{P!`uvT{&E(a3Omd-KKHmu9Dc_2~Mt>wylXE&RZ{{nHvdQ;>A~ZoPSgX*fXUrn|q!ps9Lp z&YNz*3JeZ#wQ-bVAu+e-<2rxXZy@7A0Zp%`G1#Fm%wR2g{xC>$mrZELOG;B+cX(u5 z9Di$gQN^&1%K~Z*<@PlvrQfZnkm`M1u!>|wt@L(Q*?U)`pjVQ==XzBy=9ydvLjAc7 z2h*GLMIjgGYibA^5noq)QEOSL^PD}h;#~3j8LhGv*DTX@#m;HTXPFd_X4^=i=Ii_y zhK0?j$Y8Uk!v^Ug1GMQ2huIq&@ycVEz_#$Z z+eN`Dy#_{8*^Rf|!JTfm);C_e&V=#=GQQx04Fb$avXqjO|hfg z#~8;Xje;tDS5b($mEw*^ic#CIIlc&0lHOY5vwJyR!n+?Ld7`#7hPn4iT^e=ijWqg`(o|&Z%t&$H z_kHF2ouA<kNS_B5SBFWj36vYh#$U z?rh)zxhmiVZs=p%d#GgBxPPu{)k=*?~G{`Q@j{AoR)NVRW^ zpaMj*`U_^xcEv3G)wEs=%Tn9?wD&Qb!`?rc&mT$}BVD6i!IcKf-2>b7Ea8r?Zg6E?v|8*dn+6p6M;?x|S42C8;EB>qK5 + echo "Applying CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl replace -Rf /etc/crd || kubectl create -Rf /etc/crd; + echo "Done!" + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-delete + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ .Chart.Name }}-delete + labels: + app: {{ .Chart.Name }} + spec: + serviceAccountName: {{ .Chart.Name }}-manager + securityContext: + runAsNonRoot: false + runAsUser: 0 + containers: + - name: delete-crds + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + echo "Deleting CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl delete --ignore-not-found=true -Rf /etc/crd; + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/manifest.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/manifest.yaml new file mode 100644 index 0000000000..8dc9dfb447 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/manifest.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Chart.Name }}-manifest + namespace: {{ .Release.Namespace }} +data: + crd-manifest.tgz.b64: + {{- .Files.Get "files/crd-manifest.tgz" | b64enc | indent 4 }} diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/rbac.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/rbac.yaml new file mode 100644 index 0000000000..a4d498b0fa --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/rbac.yaml @@ -0,0 +1,76 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: ['create', 'get', 'patch', 'delete', 'update', 'list'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-manager +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-manager +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/values.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/values.yaml new file mode 100644 index 0000000000..99e63600c4 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/values.yaml @@ -0,0 +1,17 @@ +# Default values for rancher-monitoring-crd. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +image: + repository: rancher/shell + tag: v0.2.1 + +nodeSelector: {} + +tolerations: [] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.editorconfig b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.editorconfig new file mode 100644 index 0000000000..f5ee2f4610 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[files/dashboards/*.json] +indent_size = 2 +indent_style = space \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.helmignore new file mode 100644 index 0000000000..9bdbec92b4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.helmignore @@ -0,0 +1,29 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# helm/charts +OWNERS +hack/ +ci/ +kube-prometheus-*.tgz + +unittests/ +files/dashboards/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CHANGELOG.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CHANGELOG.md new file mode 100644 index 0000000000..8178169b91 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog +All notable changes from the upstream Prometheus Operator chart will be added to this file. + +## [Package Version 00] - 2020-07-19 +### Added +- Added [Prometheus Adapter](https://github.com/helm/charts/tree/master/stable/prometheus-adapter) as a dependency to the upstream Prometheus Operator chart to allow users to expose custom metrics from the default Prometheus instance deployed by this chart +- Remove `prometheus-operator/cleanup-crds.yaml` and `prometheus-operator/crds.yaml` from the Prometheus Operator upstream chart in favor of just using the CRD directory to install the CRDs. +- Added support for `rkeControllerManager`, `rkeScheduler`, `rkeProxy`, and `rkeEtcd` PushProx exporters for monitoring k8s components within RKE clusters +- Added support for a `k3sServer` PushProx exporter that monitors k3s server components (`kubeControllerManager`, `kubeScheduler`, and `kubeProxy`) within k3s clusters +- Added support for `kubeAdmControllerManager`, `kubeAdmScheduler`, `kubeAdmProxy`, and `kubeAdmEtcd` PushProx exporters for monitoring k8s components within kubeAdm clusters +- Added support for `rke2ControllerManager`, `rke2Scheduler`, `rke2Proxy`, and `rke2Etcd` PushProx exporters for monitoring k8s components within rke2 clusters +- Exposed `prometheus.prometheusSpec.ignoreNamespaceSelectors` on values.yaml and set it to `false` by default. This value instructs the default Prometheus server deployed with this chart to ignore the `namespaceSelector` field within any created ServiceMonitor or PodMonitor CRs that it selects. This prevents ServiceMonitors and PodMonitors from configuring the Prometheus scrape configuration to monitor resources outside the namespace that they are deployed in; if a user needs to have one ServiceMonitor / PodMonitor monitor resources within several namespaces (such as the resources that are used to monitor Istio in a default installation), they should not enable this option since it would require them to create one ServiceMonitor / PodMonitor CR per namespace that they would like to monitor. Relevant fields were also updated in the default README.md. +- Added `grafana.sidecar.dashboards.searchNamespace` to `values.yaml` with a default value of `cattle-dashboards`. The namespace provided should contain all ConfigMaps with the label `grafana_dashboard` and will be searched by the Grafana Dashboards sidecar for updates. The namespace specified is also created along with this deployment. All default dashboard ConfigMaps have been relocated from the deployment namespace to the namespace specified +- Added `monitoring-admin`, `monitoring-edit`, and `monitoring-view` default `ClusterRoles` to allow admins to assign roles to users to interact with Prometheus Operator CRs. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `ClusterRoleBinding` to bind these roles to a Subject to allow them to set up or view `ServiceMonitors` / `PodMonitors` / `PrometheusRules` and view `Prometheus` or `Alertmanager` CRs across the cluster. If `.Values.global.rbac.userRoles.aggregateRolesForRBAC` is enabled, these ClusterRoles will aggregate into the respective default ClusterRoles provided by Kubernetes +- Added `monitoring-config-admin`, `monitoring-config-edit` and `monitoring-config-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `Secrets` and `ConfigMaps` within the `cattle-monitoring-system` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-monitoring-system` namespace to allow them to modify Secrets / ConfigMaps tied to the deployment, such as your Alertmanager Config Secret. +- Added `monitoring-dashboard-admin`, `monitoring-dashboard-edit` and `monitoring-dashboard-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `ConfigMaps` within the `cattle-dashboards` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`) and deploying Grafana as part of this chart. In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-dashboards` namespace to allow them to create / modify ConfigMaps that contain the JSON used to persist Grafana Dashboards on the cluster. +- Added default resource limits for `Prometheus Operator`, `Prometheus`, `AlertManager`, `Grafana`, `kube-state-metrics`, `node-exporter` +- Added a default template `rancher_defaults.tmpl` to AlertManager that Rancher will offer to users in order to help configure the way alerts are rendered on a notifier. Also updated the default template deployed with this chart to reference that template and added an example of a Slack config using this template as a comment in the `values.yaml`. +- Added support for private registries via introducing a new field for `global.cattle.systemDefaultRegistry` that, if supplied, will automatically be prepended onto every image used by the chart. +- Added a default `nginx` proxy container deployed with Grafana whose config is set in the `ConfigMap` located in `charts/grafana/templates/nginx-config.yaml`. The purpose of this container is to make it possible to view Grafana's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8080` (with a `portName` of `nginx-http` instead of the default `service`), which is also where the Grafana service will now point to, and will forward all requests to the Grafana container listening on the default port `3000`. +- Added a default `nginx` proxy container deployed with Prometheus whose config is set in the `ConfigMap` located in `templates/prometheus/nginx-config.yaml`. The purpose of this container is to make it possible to view Prometheus's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8081` (with a `portName` of `nginx-http` instead of the default `web`), which is also where the Prometheus service will now point to, and will forward all requests to the Prometheus container listening on the default port `9090`. +- Added support for passing CIS Scans in a hardened cluster by introducing a Job that patches the default service account within the `cattle-monitoring-system` and `cattle-dashboards` namespaces on install or upgrade and adding a default allow all `NetworkPolicy` to the `cattle-monitoring-system` and `cattle-dashboards` namespaces. +### Modified +- Updated the chart name from `prometheus-operator` to `rancher-monitoring` and added the `io.rancher.certified: rancher` annotation to `Chart.yaml` +- Modified the default `node-exporter` port from `9100` to `9796` +- Modified the default `nameOverride` to `rancher-monitoring`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Modified the default `namespaceOverride` to `cattle-monitoring-system`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Configured some default values for `grafana.service` values and exposed them in the default README.md +- The default namespaces the following ServiceMonitors were changed from the deployment namespace to allow them to continue to monitor metrics when `prometheus.prometheusSpec.ignoreNamespaceSelectors` is enabled: + - `core-dns`: `kube-system` + - `api-server`: `default` + - `kube-controller-manager`: `kube-system` + - `kubelet`: `{{ .Values.kubelet.namespace }}` +- Disabled the following deployments by default (can be enabled if required): + - `AlertManager` + - `kube-controller-manager` metrics exporter + - `kube-etcd` metrics exporter + - `kube-scheduler` metrics exporter + - `kube-proxy` metrics exporter +- Updated default Grafana `deploymentStrategy` to `Recreate` to prevent deployments from being stuck on upgrade if a PV is attached to Grafana +- Modified the default `SelectorNilUsesHelmValues` to default to `false`. As a result, we look for all CRs with any labels in all namespaces by default rather than just the ones tagged with the label `release: rancher-monitoring`. +- Modified the default images used by the `rancher-monitoring` chart to point to Rancher mirrors of the original images from upstream. +- Modified the behavior of the chart to create the Alertmanager Config Secret via a pre-install hook instead of using the normal Helm lifecycle to manage the secret. The benefit of this approach is that all changes to the Config Secret done on a live cluster will never get overridden on a `helm upgrade` since the secret only gets created on a `helm install`. If you would like the secret to be cleaned up on an `helm uninstall`, enable `alertmanager.cleanupOnUninstall`; however, this is disabled by default to prevent the loss of alerting configuration on an uninstall. This secret will never be modified on a `helm upgrade`. +- Modified the default `securityContext` for `Pod` templates across the chart to `{"runAsNonRoot": "true", "runAsUser": "1000"}` and replaced `grafana.rbac.pspUseAppArmor` in favor of `grafana.rbac.pspAnnotations={}` in order to make it possible to deploy this chart on a hardened cluster which does not support Seccomp or AppArmor annotations in PSPs. Users can always choose to specify the annotations they want to use for the PSP directly as part of the values provided. +- Modified `.Values.prometheus.prometheusSpec.containers` to take in a string representing a template that should be rendered by Helm (via `tpl`) instead of allowing a user to provide YAML directly. +- Modified the default Grafana configuration to auto assign users who access Grafana to the Viewer role and enable anonymous access to Grafana dashboards by default. This default works well for a Rancher user who is accessing Grafana via the `kubectl proxy` on the Rancher Dashboard UI since anonymous users who enter via the proxy are authenticated by the k8s API Server, but you can / should modify this behavior if you plan on exposing Grafana in a way that does not require authentication (e.g. as a `NodePort` service). +- Modified the default Grafana configuration to add a default dashboard for Rancher on the Grafana home page. \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CONTRIBUTING.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CONTRIBUTING.md new file mode 100644 index 0000000000..f6ce2a3235 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contributing Guidelines + +## How to contribute to this chart + +1. Fork this repository, develop and test your Chart. +1. Bump the chart version for every change. +1. Ensure PR title has the prefix `[kube-prometheus-stack]` +1. When making changes to rules or dashboards, see the README.md section on how to sync data from upstream repositories +1. Check the `hack/minikube` folder has scripts to set up minikube and components of this chart that will allow all components to be scraped. You can use this configuration when validating your changes. +1. Check for changes of RBAC rules. +1. Check for changes in CRD specs. +1. PR must pass the linter (`helm lint`) diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/Chart.yaml new file mode 100644 index 0000000000..2aa440a1ca --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/Chart.yaml @@ -0,0 +1,126 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + - name: Upstream Project + url: https://github.com/prometheus-operator/kube-prometheus + artifacthub.io/operator: "true" + catalog.cattle.io/auto-install: rancher-monitoring-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/deploys-on-os: windows + catalog.cattle.io/display-name: Monitoring + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: monitoring.coreos.com.prometheus/v1 + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: rancher-monitoring + catalog.cattle.io/requests-cpu: 4500m + catalog.cattle.io/requests-memory: 4000Mi + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: monitoring + catalog.cattle.io/upstream-version: 57.0.3 +apiVersion: v2 +appVersion: v0.72.0 +dependencies: +- condition: grafana.enabled + name: grafana + repository: file://./charts/grafana +- condition: hardenedKubelet.enabled + name: hardenedKubelet + repository: file://./charts/hardenedKubelet +- condition: hardenedNodeExporter.enabled + name: hardenedNodeExporter + repository: file://./charts/hardenedNodeExporter +- condition: k3sServer.enabled + name: k3sServer + repository: file://./charts/k3sServer +- condition: kubeStateMetrics.enabled + name: kube-state-metrics + repository: file://./charts/kube-state-metrics +- condition: kubeAdmControllerManager.enabled + name: kubeAdmControllerManager + repository: file://./charts/kubeAdmControllerManager +- condition: kubeAdmEtcd.enabled + name: kubeAdmEtcd + repository: file://./charts/kubeAdmEtcd +- condition: kubeAdmProxy.enabled + name: kubeAdmProxy + repository: file://./charts/kubeAdmProxy +- condition: kubeAdmScheduler.enabled + name: kubeAdmScheduler + repository: file://./charts/kubeAdmScheduler +- condition: prometheus-adapter.enabled + name: prometheus-adapter + repository: file://./charts/prometheus-adapter +- condition: nodeExporter.enabled + name: prometheus-node-exporter + repository: file://./charts/prometheus-node-exporter +- condition: rke2ControllerManager.enabled + name: rke2ControllerManager + repository: file://./charts/rke2ControllerManager +- condition: rke2Etcd.enabled + name: rke2Etcd + repository: file://./charts/rke2Etcd +- condition: rke2IngressNginx.enabled + name: rke2IngressNginx + repository: file://./charts/rke2IngressNginx +- condition: rke2Proxy.enabled + name: rke2Proxy + repository: file://./charts/rke2Proxy +- condition: rke2Scheduler.enabled + name: rke2Scheduler + repository: file://./charts/rke2Scheduler +- condition: rkeControllerManager.enabled + name: rkeControllerManager + repository: file://./charts/rkeControllerManager +- condition: rkeEtcd.enabled + name: rkeEtcd + repository: file://./charts/rkeEtcd +- condition: rkeIngressNginx.enabled + name: rkeIngressNginx + repository: file://./charts/rkeIngressNginx +- condition: rkeProxy.enabled + name: rkeProxy + repository: file://./charts/rkeProxy +- condition: rkeScheduler.enabled + name: rkeScheduler + repository: file://./charts/rkeScheduler +- condition: windowsExporter.enabled + name: windowsExporter + repository: file://./charts/windowsExporter +description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, + and Prometheus rules combined with documentation and scripts to provide easy to + operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus + Operator. +home: https://github.com/prometheus-operator/kube-prometheus +icon: file://assets/logos/rancher-monitoring.png +keywords: +- operator +- prometheus +- kube-prometheus +kubeVersion: '>=1.19.0-0' +maintainers: +- email: andrew@quadcorps.co.uk + name: andrewgkew +- email: gianrubio@gmail.com + name: gianrubio +- email: github.gkarthiks@gmail.com + name: gkarthiks +- email: kube-prometheus-stack@sisti.pt + name: GMartinez-Sisti +- email: github@jkroepke.de + name: jkroepke +- email: scott@r6by.com + name: scottrigby +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: quentin.bisson@gmail.com + name: QuentinBisson +name: rancher-monitoring +sources: +- https://github.com/prometheus-community/helm-charts +- https://github.com/prometheus-operator/kube-prometheus +type: application +version: 104.1.2-rc.1+up57.0.3 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/README.md new file mode 100644 index 0000000000..9baf58bb16 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/README.md @@ -0,0 +1,1080 @@ +# kube-prometheus-stack + +Installs the [kube-prometheus stack](https://github.com/prometheus-operator/kube-prometheus), a collection of Kubernetes manifests, [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with [Prometheus](https://prometheus.io/) using the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). + +See the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) README for details about components, dashboards, and alerts. + +_Note: This chart was formerly named `prometheus-operator` chart, now renamed to more clearly reflect that it installs the `kube-prometheus` project stack, within which Prometheus Operator is only one component._ + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3+ + +## Get Helm Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Dependencies + +By default this chart installs additional, dependent charts: + +- [prometheus-community/kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) +- [prometheus-community/prometheus-node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) +- [grafana/grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) + +To disable dependencies during installation, see [multiple releases](#multiple-releases) below. + +_See [helm dependency](https://helm.sh/docs/helm/helm_dependency/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +CRDs created by this chart are not removed by default and should be manually cleaned up: + +```console +kubectl delete crd alertmanagerconfigs.monitoring.coreos.com +kubectl delete crd alertmanagers.monitoring.coreos.com +kubectl delete crd podmonitors.monitoring.coreos.com +kubectl delete crd probes.monitoring.coreos.com +kubectl delete crd prometheusagents.monitoring.coreos.com +kubectl delete crd prometheuses.monitoring.coreos.com +kubectl delete crd prometheusrules.monitoring.coreos.com +kubectl delete crd scrapeconfigs.monitoring.coreos.com +kubectl delete crd servicemonitors.monitoring.coreos.com +kubectl delete crd thanosrulers.monitoring.coreos.com +``` + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +With Helm v3, CRDs created by this chart are not updated by default and should be manually updated. +Consult also the [Helm Documentation on CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions). + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. + +### From 56.x to 57.x + +This version upgrades Prometheus-Operator to v0.72.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 55.x to 56.x + +This version upgrades Prometheus-Operator to v0.71.0, Prometheus to 2.49.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 54.x to 55.x + +This version upgrades Prometheus-Operator to v0.70.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 53.x to 54.x + +Grafana Helm Chart has bumped to version 7 + +Please note Grafana Helm Chart [changelog](https://github.com/grafana/helm-charts/tree/main/charts/grafana#to-700). + +### From 52.x to 53.x + +This version upgrades Prometheus-Operator to v0.69.1, Prometheus to 2.47.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 51.x to 52.x + +This includes the ability to select between using existing secrets or create new secret objects for various thanos config. The defaults have not changed but if you were setting: + +- `thanosRuler.thanosRulerSpec.alertmanagersConfig` or +- `thanosRuler.thanosRulerSpec.objectStorageConfig` or +- `thanosRuler.thanosRulerSpec.queryConfig` or +- `prometheus.prometheusSpec.thanos.objectStorageConfig` + +you will have to need to set `existingSecret` or `secret` based on your requirement + +For instance, the `thanosRuler.thanosRulerSpec.alertmanagersConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + secret: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +or the `thanosRuler.thanosRulerSpec.objectStorageConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + existingSecret: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +### From 50.x to 51.x + +This version upgrades Prometheus-Operator to v0.68.0, Prometheus to 2.47.0 and Thanos to v0.32.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 49.x to 50.x + +This version requires Kubernetes 1.19+. + +We do not expect any breaking changes in this version. + +### From 48.x to 49.x + +This version upgrades Prometheus-Operator to v0.67.1, 0, Alertmanager to v0.26.0, Prometheus to 2.46.0 and Thanos to v0.32.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 47.x to 48.x + +This version moved all CRDs into a dedicated sub-chart. No new CRDs are introduced in this version. +See [#3548](https://github.com/prometheus-community/helm-charts/issues/3548) for more context. + +We do not expect any breaking changes in this version. + +### From 46.x to 47.x + +This version upgrades Prometheus-Operator to v0.66.0 with new CRDs (PrometheusAgent and ScrapeConfig). + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 45.x to 46.x + +This version upgrades Prometheus-Operator to v0.65.1 with new CRDs (PrometheusAgent and ScrapeConfig), Prometheus to v2.44.0 and Thanos to v0.31.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 44.x to 45.x + +This version upgrades Prometheus-Operator to v0.63.0, Prometheus to v2.42.0 and Thanos to v0.30.2. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 43.x to 44.x + +This version upgrades Prometheus-Operator to v0.62.0, Prometheus to v2.41.0 and Thanos to v0.30.1. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +If you have explicitly set `prometheusOperator.admissionWebhooks.failurePolicy`, this value is now always used even when `.prometheusOperator.admissionWebhooks.patch.enabled` is `true` (the default). + +The values for `prometheusOperator.image.tag` & `prometheusOperator.prometheusConfigReloader.image.tag` are now empty by default and the Chart.yaml `appVersion` field is used instead. + +### From 42.x to 43.x + +This version upgrades Prometheus-Operator to v0.61.1, Prometheus to v2.40.5 and Thanos to v0.29.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 41.x to 42.x + +This includes the overridability of container registry for all containers at the global level using `global.imageRegistry` or per container image. The defaults have not changed but if you were using a custom image, you will have to override the registry of said custom container image before you upgrade. + +For instance, the prometheus-config-reloader used to be configured as follow: + +```yaml + image: + repository: quay.io/prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +But it now moved to: + +```yaml + image: + registry: quay.io + repository: prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +### From 40.x to 41.x + +This version upgrades Prometheus-Operator to v0.60.1, Prometheus to v2.39.1 and Thanos to v0.28.1. +This version also upgrades the Helm charts of kube-state-metrics to 4.20.2, prometheus-node-exporter to 4.3.0 and Grafana to 6.40.4. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +This version splits kubeScheduler recording and altering rules in separate config values. +Instead of `defaultRules.rules.kubeScheduler` the 2 new variables `defaultRules.rules.kubeSchedulerAlerting` and `defaultRules.rules.kubeSchedulerRecording` are used. + +### From 39.x to 40.x + +This version upgrades Prometheus-Operator to v0.59.1, Prometheus to v2.38.0, kube-state-metrics to v2.6.0 and Thanos to v0.28.0. +This version also upgrades the Helm charts of kube-state-metrics to 4.18.0 and prometheus-node-exporter to 4.2.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +Starting from prometheus-node-exporter version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i kube-prometheus-stack prometheus-community/kube-prometheus-stack +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 38.x to 39.x + +This upgraded prometheus-operator to v0.58.0 and prometheus to v2.37.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 37.x to 38.x + +Reverted one of the default metrics relabelings for cAdvisor added in 36.x, due to it breaking container_network_* and various other statistics. If you do not want this change, you will need to override the `kubelet.cAdvisorMetricRelabelings`. + +### From 36.x to 37.x + +This includes some default metric relabelings for cAdvisor and apiserver metrics to reduce cardinality. If you do not want these defaults, you will need to override the `kubeApiServer.metricRelabelings` and or `kubelet.cAdvisorMetricRelabelings`. + +### From 35.x to 36.x + +This upgraded prometheus-operator to v0.57.0 and prometheus to v2.36.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 34.x to 35.x + +This upgraded prometheus-operator to v0.56.0 and prometheus to v2.35.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 33.x to 34.x + +This upgrades to prometheus-operator to v0.55.0 and prometheus to v2.33.5. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 32.x to 33.x + +This upgrades the prometheus-node-exporter Chart to v3.0.0. Please review the changes to this subchart if you make customizations to hostMountPropagation. + +### From 31.x to 32.x + +This upgrades to prometheus-operator to v0.54.0 and prometheus to v2.33.1. It also changes the default for `grafana.serviceMonitor.enabled` to `true. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 30.x to 31.x + +This version removes the built-in grafana ServiceMonitor and instead relies on the ServiceMonitor of the sub-chart. +`grafana.serviceMonitor.enabled` must be set instead of `grafana.serviceMonitor.selfMonitor` and the old ServiceMonitor may +need to be manually cleaned up after deploying the new release. + +### From 29.x to 30.x + +This version updates kube-state-metrics to 4.3.0 and uses the new option `kube-state-metrics.releaseLabel=true` which adds the "release" label to kube-state-metrics labels, making scraping of the metrics by kube-prometheus-stack work out of the box again, independent of the used kube-prometheus-stack release name. If you already set the "release" label via `kube-state-metrics.customLabels` you might have to remove that and use it via the new option. + +### From 28.x to 29.x + +This version makes scraping port for kube-controller-manager and kube-scheduler dynamic to reflect changes to default serving ports +for those components in Kubernetes versions v1.22 and v1.23 respectively. + +If you deploy on clusters using version v1.22+, kube-controller-manager will be scraped over HTTPS on port 10257. + +If you deploy on clusters running version v1.23+, kube-scheduler will be scraped over HTTPS on port 10259. + +### From 27.x to 28.x + +This version disables PodSecurityPolicies by default because they are deprecated in Kubernetes 1.21 and will be removed in Kubernetes 1.25. + +If you are using PodSecurityPolicies you can enable the previous behaviour by setting `kube-state-metrics.podSecurityPolicy.enabled`, `prometheus-node-exporter.rbac.pspEnabled`, `grafana.rbac.pspEnabled` and `global.rbac.pspEnabled` to `true`. + +### From 26.x to 27.x + +This version splits prometheus-node-exporter chart recording and altering rules in separate config values. +Instead of `defaultRules.rules.node` the 2 new variables `defaultRules.rules.nodeExporterAlerting` and `defaultRules.rules.nodeExporterRecording` are used. + +Also the following defaultRules.rules has been removed as they had no effect: `kubeApiserverError`, `kubePrometheusNodeAlerting`, `kubernetesAbsent`, `time`. + +The ability to set a rubookUrl via `defaultRules.rules.rubookUrl` was reintroduced. + +### From 25.x to 26.x + +This version enables the prometheus-node-exporter subchart servicemonitor by default again, by setting `prometheus-node-exporter.prometheus.monitor.enabled` to `true`. + +### From 24.x to 25.x + +This version upgrade to prometheus-operator v0.53.1. It removes support for setting a runbookUrl, since the upstream format for runbooks changed. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 23.x to 24.x + +The custom `ServiceMonitor` for the _kube-state-metrics_ & _prometheus-node-exporter_ charts have been removed in favour of the built-in sub-chart `ServiceMonitor`; for both sub-charts this means that `ServiceMonitor` customisations happen via the values passed to the chart. If you haven't directly customised this behaviour then there are no changes required to upgrade, but if you have please read the following. + +For _kube-state-metrics_ the `ServiceMonitor` customisation is now set via `kube-state-metrics.prometheus.monitor` and the `kubeStateMetrics.serviceMonitor.selfMonitor.enabled` value has moved to `kube-state-metrics.selfMonitor.enabled`. + +For _prometheus-node-exporter_ the `ServiceMonitor` customisation is now set via `prometheus-node-exporter.prometheus.monitor` and the `nodeExporter.jobLabel` values has moved to `prometheus-node-exporter.prometheus.monitor.jobLabel`. + +### From 22.x to 23.x + +Port names have been renamed for Istio's +[explicit protocol selection](https://istio.io/latest/docs/ops/configuration/traffic-management/protocol-selection/#explicit-protocol-selection). + +| | old value | new value | +|-|-----------|-----------| +| `alertmanager.alertmanagerSpec.portName` | `web` | `http-web` | +| `grafana.service.portName` | `service` | `http-web` | +| `prometheus-node-exporter.service.portName` | `metrics` (hardcoded) | `http-metrics` | +| `prometheus.prometheusSpec.portName` | `web` | `http-web` | + +### From 21.x to 22.x + +Due to the upgrade of the `kube-state-metrics` chart, removal of its deployment/stateful needs to done manually prior to upgrading: + +```console +kubectl delete deployments.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +or if you use autosharding: + +```console +kubectl delete statefulsets.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +### From 20.x to 21.x + +The config reloader values have been refactored. All the values have been moved to the key `prometheusConfigReloader` and the limits and requests can now be set separately. + +### From 19.x to 20.x + +Version 20 upgrades prometheus-operator from 0.50.x to 0.52.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 18.x to 19.x + +`kubeStateMetrics.serviceMonitor.namespaceOverride` was removed. +Please use `kube-state-metrics.namespaceOverride` instead. + +### From 17.x to 18.x + +Version 18 upgrades prometheus-operator from 0.49.x to 0.50.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 16.x to 17.x + +Version 17 upgrades prometheus-operator from 0.48.x to 0.49.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 15.x to 16.x + +Version 16 upgrades kube-state-metrics to v2.0.0. This includes changed command-line arguments and removed metrics, see this [blog post](https://kubernetes.io/blog/2021/04/13/kube-state-metrics-v-2-0/). This version also removes Grafana dashboards that supported Kubernetes 1.14 or earlier. + +### From 14.x to 15.x + +Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 13.x to 14.x + +Version 14 upgrades prometheus-operator from 0.45.x to 0.46.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 12.x to 13.x + +Version 13 upgrades prometheus-operator from 0.44.x to 0.45.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +``` + +### From 11.x to 12.x + +Version 12 upgrades prometheus-operator from 0.43.x to 0.44.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.44/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +``` + +The chart was migrated to support only helm v3 and later. + +### From 10.x to 11.x + +Version 11 upgrades prometheus-operator from 0.42.x to 0.43.x. Starting with 0.43.x an additional `AlertmanagerConfigs` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.43/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +``` + +Version 11 removes the deprecated tlsProxy via ghostunnel in favor of native TLS support the prometheus-operator gained with v0.39.0. + +### From 9.x to 10.x + +Version 10 upgrades prometheus-operator from 0.38.x to 0.42.x. Starting with 0.40.x an additional `Probes` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +``` + +### From 8.x to 9.x + +Version 9 of the helm chart removes the existing `additionalScrapeConfigsExternal` in favour of `additionalScrapeConfigsSecret`. This change lets users specify the secret name and secret key to use for the additional scrape configuration of prometheus. This is useful for users that have prometheus-operator as a subchart and also have a template that creates the additional scrape configuration. + +### From 7.x to 8.x + +Due to new template functions being used in the rules in version 8.x.x of the chart, an upgrade to Prometheus Operator and Prometheus is necessary in order to support them. First, upgrade to the latest version of 7.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version 7.5.0 +``` + +Then upgrade to 8.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version [8.x.x] +``` + +Minimal recommended Prometheus version for this chart release is `2.12.x` + +### From 6.x to 7.x + +Due to a change in grafana subchart, version 7.x.x now requires Helm >= 2.12.0. + +### From 5.x to 6.x + +Due to a change in deployment labels of kube-state-metrics, the upgrade requires `helm upgrade --force` in order to re-create the deployment. If this is not done an error will occur indicating that the deployment cannot be modified: + +```console +invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/name":"kube-state-metrics"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +If this error has already been encountered, a `helm history` command can be used to determine which release has worked, then `helm rollback` to the release, then `helm upgrade --force` to this new one + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-prometheus-stack +``` + +You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. + +### Rancher Monitoring Configuration + +The following table shows values exposed by Rancher Monitoring's additions to the chart: + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `nameOverride` | Provide a name that should be used instead of the chart name when naming all resources deployed by this chart |`"rancher-monitoring"`| +| `namespaceOverride` | Override the deployment namespace | `"cattle-monitoring-system"` | +| `global.rbac.userRoles.create` | Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets | `true` | +| `global.rbac.userRoles.aggregateToDefaultRoles` | Aggregate default user ClusterRoles into default k8s ClusterRoles | `true` | +| `prometheus-adapter.enabled` | Whether to install [prometheus-adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) within the cluster | `true` | +| `prometheus-adapter.prometheus.url` | A URL pointing to the Prometheus deployment within your cluster. The default value is set based on the assumption that you plan to deploy the default Prometheus instance from this chart where `.Values.namespaceOverride=cattle-monitoring-system` and `.Values.nameOverride=rancher-monitoring` | `http://rancher-monitoring-prometheus.cattle-monitoring-system.svc` | +| `prometheus-adapter.prometheus.port` | The port on the Prometheus deployment that Prometheus Adapter can make requests to | `9090` | +| `prometheus.prometheusSpec.ignoreNamespaceSelectors` | Ignore NamespaceSelector settings from the PodMonitor and ServiceMonitor configs. If true, PodMonitors and ServiceMonitors can only discover Pods and Services within the namespace they are deployed into | `false` | + +The following values are enabled for different distributions via [rancher-pushprox](https://github.com/rancher/dev-charts/tree/master/packages/rancher-pushprox). See the rancher-pushprox `README.md` for more information on what all values can be configured for the PushProxy chart. + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `rkeControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in RKE clusters | `false` | +| `rkeScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in RKE clusters | `false` | +| `rkeProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in RKE clusters | `false` | +| `rkeIngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE clusters | `false` | +| `rkeEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in RKE clusters | `false` | +| `rke2IngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE2 clusters | `false` | +| `k3sServer.enabled` | Create a PushProx installation for monitoring k3s-server metrics (accounts for kube-controller-manager, kube-scheduler, and kube-proxy metrics) in k3s clusters | `false` | +| `kubeAdmControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in kubeAdm clusters | `false` | +| `kubeAdmScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in kubeAdm clusters | `false` | +| `kubeAdmProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in kubeAdm clusters | `false` | +| `kubeAdmEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in kubeAdm clusters | `false` | + + +### Multiple releases + +The same chart can be used to run multiple Prometheus instances in the same cluster if required. To achieve this, it is necessary to run only one instance of prometheus-operator and a pair of alertmanager pods for an HA configuration, while all other components need to be disabled. To disable a dependency during installation, set `kubeStateMetrics.enabled`, `nodeExporter.enabled` and `grafana.enabled` to `false`. + +## Work-Arounds for Known Issues + +### Running on private GKE clusters + +When Google configure the control plane for private clusters, they automatically configure VPC peering between your Kubernetes cluster’s network and a separate Google managed project. In order to restrict what Google are able to access within your cluster, the firewall rules configured restrict access to your Kubernetes pods. This means that in order to use the webhook component with a GKE private cluster, you must configure an additional firewall rule to allow the GKE control plane access to your webhook pod. + +You can read more information on how to add firewall rules for the GKE control plane nodes in the [GKE docs](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) + +Alternatively, you can disable the hooks by setting `prometheusOperator.admissionWebhooks.enabled=false`. + +## PrometheusRules Admission Webhooks + +With Prometheus Operator version 0.30+, the core Prometheus Operator pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent malformed rules from being added to the cluster. + +### How the Chart Configures the Hooks + +A validating and mutating webhook configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. + +1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. +2. The prometheus operator pod is configured to use a TLS proxy container, which will load that certificate. +3. Validating and Mutating webhook configurations are created in the cluster, with their failure mode set to Ignore. This allows rules to be created by the same chart at the same time, even though the webhook has not yet been fully set up - it does not have the correct CA field set. +4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations + +### Alternatives + +It should be possible to use [jetstack/cert-manager](https://github.com/jetstack/cert-manager) if a more complete solution is required, but it has not been tested. + +You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `prometheusOperator.admissionWebhooks.certManager.enabled` value to true. + +### Limitations + +Because the operator can only run as a single pod, there is potential for this component failure to cause rule deployment failure. Because this risk is outweighed by the benefit of having validation, the feature is enabled by default. + +## Developing Prometheus Rules and Grafana Dashboards + +This chart Grafana Dashboards and Prometheus Rules are just a copy from [prometheus-operator/prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) and other sources, synced (with alterations) by scripts in [hack](hack) folder. In order to introduce any changes you need to first [add them to the original repository](https://github.com/prometheus-operator/kube-prometheus/blob/main/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md) and then sync there by scripts. + +## Further Information + +For more in-depth documentation of configuration options meanings, please see + +- [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) +- [Prometheus](https://prometheus.io/docs/introduction/overview/) +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana#grafana-helm-chart) + +## prometheus.io/scrape + +The prometheus operator does not support annotation-based discovery of services, using the `PodMonitor` or `ServiceMonitor` CRD in its place as they provide far more configuration options. +For information on how to use PodMonitors/ServiceMonitors, please see the documentation on the `prometheus-operator/prometheus-operator` documentation here: + +- [ServiceMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-servicemonitors) +- [PodMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-podmonitors) +- [Running Exporters](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/running-exporters.md) + +By default, Prometheus discovers PodMonitors and ServiceMonitors within its namespace, that are labeled with the same release tag as the prometheus-operator release. +Sometimes, you may need to discover custom PodMonitors/ServiceMonitors, for example used to scrape data from third-party applications. +An easy way of doing this, without compromising the default PodMonitors/ServiceMonitors discovery, is allowing Prometheus to discover all PodMonitors/ServiceMonitors within its namespace, without applying label filtering. +To do so, you can set `prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues` and `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues` to `false`. + +## Migrating from stable/prometheus-operator chart + +## Zero downtime + +Since `kube-prometheus-stack` is fully compatible with the `stable/prometheus-operator` chart, a migration without downtime can be achieved. +However, the old name prefix needs to be kept. If you want the new name please follow the step by step guide below (with downtime). + +You can override the name to achieve this: + +```console +helm upgrade prometheus-operator prometheus-community/kube-prometheus-stack -n monitoring --reuse-values --set nameOverride=prometheus-operator +``` + +**Note**: It is recommended to run this first with `--dry-run --debug`. + +## Redeploy with new name (downtime) + +If the **prometheus-operator** values are compatible with the new **kube-prometheus-stack** chart, please follow the below steps for migration: + +> The guide presumes that chart is deployed in `monitoring` namespace and the deployments are running there. If in other namespace, please replace the `monitoring` to the deployed namespace. + +1. Patch the PersistenceVolume created/used by the prometheus-operator chart to `Retain` claim policy: + + ```console + kubectl patch pv/ -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + ``` + + **Note:** To execute the above command, the user must have a cluster wide permission. Please refer [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +2. Uninstall the **prometheus-operator** release and delete the existing PersistentVolumeClaim, and verify PV become Released. + + ```console + helm uninstall prometheus-operator -n monitoring + kubectl delete pvc/ -n monitoring + ``` + + Additionally, you have to manually remove the remaining `prometheus-operator-kubelet` service. + + ```console + kubectl delete service/prometheus-operator-kubelet -n kube-system + ``` + + You can choose to remove all your existing CRDs (ServiceMonitors, Podmonitors, etc.) if you want to. + +3. Remove current `spec.claimRef` values to change the PV's status from Released to Available. + + ```console + kubectl patch pv/ --type json -p='[{"op": "remove", "path": "/spec/claimRef"}]' -n monitoring + ``` + +**Note:** To execute the above command, the user must have a cluster wide permission. Please refer to [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +After these steps, proceed to a fresh **kube-prometheus-stack** installation and make sure the current release of **kube-prometheus-stack** matching the `volumeClaimTemplate` values in the `values.yaml`. + +The binding is done via matching a specific amount of storage requested and with certain access modes. + +For example, if you had storage specified as this with **prometheus-operator**: + +```yaml +volumeClaimTemplate: + spec: + storageClassName: gp2 + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi +``` + +You have to specify matching `volumeClaimTemplate` with 50Gi storage and `ReadWriteOnce` access mode. + +Additionally, you should check the current AZ of your legacy installation's PV, and configure the fresh release to use the same AZ as the old one. If the pods are in a different AZ than the PV, the release will fail to bind the existing one, hence creating a new PV. + +This can be achieved either by specifying the labels through `values.yaml`, e.g. setting `prometheus.prometheusSpec.nodeSelector` to: + +```yaml +nodeSelector: + failure-domain.beta.kubernetes.io/zone: east-west-1a +``` + +or passing these values as `--set` overrides during installation. + +The new release should now re-attach your previously released PV with its content. + +## Migrating from coreos/prometheus-operator chart + +The multiple charts have been combined into a single chart that installs prometheus operator, prometheus, alertmanager, grafana as well as the multitude of exporters necessary to monitor a cluster. + +There is no simple and direct migration path between the charts as the changes are extensive and intended to make the chart easier to support. + +The capabilities of the old chart are all available in the new chart, including the ability to run multiple prometheus instances on a single cluster - you will need to disable the parts of the chart you do not wish to deploy. + +You can check out the tickets for this change [here](https://github.com/prometheus-operator/prometheus-operator/issues/592) and [here](https://github.com/helm/charts/pull/6765). + +### High-level overview of Changes + +#### Added dependencies + +The chart has added 3 [dependencies](#dependencies). + +- Node-Exporter, Kube-State-Metrics: These components are loaded as dependencies into the chart, and are relatively simple components +- Grafana: The Grafana chart is more feature-rich than this chart - it contains a sidecar that is able to load data sources and dashboards from configmaps deployed into the same cluster. For more information check out the [documentation for the chart](https://github.com/grafana/helm-charts/blob/main/charts/grafana/README.md) + +#### Kubelet Service + +Because the kubelet service has a new name in the chart, make sure to clean up the old kubelet service in the `kube-system` namespace to prevent counting container metrics twice. + +#### Persistent Volumes + +If you would like to keep the data of the current persistent volumes, it should be possible to attach existing volumes to new PVCs and PVs that are created using the conventions in the new chart. For example, in order to use an existing Azure disk for a helm release called `prometheus-migration` the following resources can be created: + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc-prometheus-migration-prometheus-0 +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: pvc-prometheus-migration-prometheus-0 + diskURI: /subscriptions/f5125d82-2622-4c50-8d25-3f7ba3e9ac4b/resourceGroups/sample-migration-resource-group/providers/Microsoft.Compute/disks/pvc-prometheus-migration-prometheus-0 + fsType: "" + kind: Managed + readOnly: false + capacity: + storage: 1Gi + persistentVolumeReclaimPolicy: Delete + storageClassName: prometheus + volumeMode: Filesystem +``` + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: prometheus + prometheus: prometheus-migration-prometheus + name: prometheus-prometheus-migration-prometheus-db-prometheus-prometheus-migration-prometheus-0 + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: prometheus + volumeMode: Filesystem + volumeName: pvc-prometheus-migration-prometheus-0 +``` + +The PVC will take ownership of the PV and when you create a release using a persistent volume claim template it will use the existing PVCs as they match the naming convention used by the chart. For other cloud providers similar approaches can be used. + +#### KubeProxy + +The metrics bind address of kube-proxy is default to `127.0.0.1:10249` that prometheus instances **cannot** access to. You should expose metrics by changing `metricsBindAddress` field value to `0.0.0.0:10249` if you want to collect them. + +Depending on the cluster, the relevant part `config.conf` will be in ConfigMap `kube-system/kube-proxy` or `kube-system/kube-proxy-config`. For example: + +```console +kubectl -n kube-system edit cm kube-proxy +``` + +```yaml +apiVersion: v1 +data: + config.conf: |- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + # ... + # metricsBindAddress: 127.0.0.1:10249 + metricsBindAddress: 0.0.0.0:10249 + # ... + kubeconfig.conf: |- + # ... +kind: ConfigMap +metadata: + labels: + app: kube-proxy + name: kube-proxy + namespace: kube-system +``` diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/app-README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/app-README.md new file mode 100644 index 0000000000..3920854384 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/app-README.md @@ -0,0 +1,46 @@ +# Rancher Monitoring and Alerting + + This chart is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) chart. The chart deploys [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) and its CRDs along with [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana), [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) and additional charts / Kubernetes manifests to gather metrics. It allows users to monitor their Kubernetes clusters, view metrics in Grafana dashboards, and set up alerts and notifications. + +For more information on how to use the feature, refer to our [docs](https://rancher.com/docs/rancher/v2.x/en/monitoring-alerting/v2.5/). + +The chart installs the following components: + +- [Prometheus Operator](https://github.com/coreos/prometheus-operator) - The operator provides easy monitoring definitions for Kubernetes services, manages [Prometheus](https://prometheus.io/) and [AlertManager](https://prometheus.io/docs/alerting/latest/alertmanager/) instances, and adds default scrape targets for some Kubernetes components. +- [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) - A collection of community-curated Kubernetes manifests, Grafana Dashboards, and PrometheusRules that deploy a default end-to-end cluster monitoring configuration. +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) - Grafana allows a user to create / view dashboards based on the cluster metrics collected by Prometheus. +- [node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) / [kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) / [rancher-pushprox](https://github.com/rancher/charts/tree/dev-v2.7/packages/rancher-monitoring/rancher-pushprox/charts) - These charts monitor various Kubernetes components across different Kubernetes cluster types. +- [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) - The adapter allows a user to expose custom metrics, resource metrics, and external metrics on the default [Prometheus](https://prometheus.io/) instance to the Kubernetes API Server. + +For more information, review the Helm README of this chart. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Upgrading from 100.0.0+up16.6.0 to 100.1.0+up19.0.3 + +### Noticeable changes: +Grafana: +- `sidecar.dashboards.searchNamespace`, `sidecar.datasources.searchNamespace` and `sidecar.notifiers.searchNamespace` support a list of namespaces now. + +Kube-state-metrics +- the type of `collectors` is changed from Dictionary to List. +- `kubeStateMetrics.serviceMonitor.namespaceOverride` was replaced by `kube-state-metrics.namespaceOverride`. + +### Known issues: +- Occasionally, the upgrade fails with errors related to the webhook `prometheusrulemutate.monitoring.coreos.com`. This is a known issue in the upstream, and the workaround is to trigger the upgrade one more time. [32416](https://github.com/rancher/rancher/issues/32416#issuecomment-828881726) diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/.helmignore new file mode 100644 index 0000000000..8cade1318f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.vscode +.project +.idea/ +*.tmproj +OWNERS diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/Chart.yaml new file mode 100644 index 0000000000..ff6bcb26aa --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/Chart.yaml @@ -0,0 +1,39 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/grafana/helm-charts + - name: Upstream Project + url: https://github.com/grafana/grafana + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-grafana +apiVersion: v2 +appVersion: 10.4.1 +description: The leading tool for querying and visualizing time series and metrics. +home: https://grafana.com +icon: https://artifacthub.io/image/b4fed1a7-6c8f-4945-b99d-096efa3e4116 +keywords: +- monitoring +- metric +kubeVersion: '>=1.26.0-0' +maintainers: +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rluckie@cisco.com + name: rtluckie +- email: maor.friedman@redhat.com + name: maorfr +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: mail@torstenwalter.de + name: torstenwalter +name: grafana +sources: +- https://github.com/grafana/grafana +- https://github.com/grafana/helm-charts +type: application +version: 7.3.11 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/README.md new file mode 100644 index 0000000000..0ff07f297d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/README.md @@ -0,0 +1,770 @@ +# Grafana Helm Chart + +* Installs the web dashboarding system [Grafana](http://grafana.org/) + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 4.0.0 (And 3.12.1) + +This version requires Helm >= 2.12.0. + +### To 5.0.0 + +You have to add --force to your helm upgrade command as the labels of the chart have changed. + +### To 6.0.0 + +This version requires Helm >= 3.1.0. + +### To 7.0.0 + +For consistency with other Helm charts, the `global.image.registry` parameter was renamed +to `global.imageRegistry`. If you were not previously setting `global.image.registry`, no action +is required on upgrade. If you were previously setting `global.image.registry`, you will +need to instead set `global.imageRegistry`. + +## Configuration + +| Parameter | Description | Default | +|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------| +| `replicas` | Number of nodes | `1` | +| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` | +| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` | +| `podDisruptionBudget.apiVersion` | Pod disruption apiVersion | `nil` | +| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` | +| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` | +| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`| +| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `image.registry` | Image registry | `docker.io` | +| `image.repository` | Image repository | `grafana/grafana` | +| `image.tag` | Overrides the Grafana image tag whose default is the chart appVersion (`Must be >= 5.0.0`) | `` | +| `image.sha` | Image sha (optional) | `` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets (can be templated) | `[]` | +| `service.enabled` | Enable grafana service | `true` | +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Kubernetes port where service is exposed | `80` | +| `service.portName` | Name of the port on the service | `service` | +| `service.appProtocol` | Adds the appProtocol field to the service | `` | +| `service.targetPort` | Internal service is port | `3000` | +| `service.nodePort` | Kubernetes service nodePort | `nil` | +| `service.annotations` | Service annotations (can be templated) | `{}` | +| `service.labels` | Custom labels | `{}` | +| `service.clusterIP` | internal cluster service IP | `nil` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | +| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | +| `service.externalIPs` | service external IP addresses | `[]` | +| `service.externalTrafficPolicy` | change the default externalTrafficPolicy | `nil` | +| `headlessService` | Create a headless service | `false` | +| `extraExposePorts` | Additional service ports for sidecar containers| `[]` | +| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.annotations` | Ingress annotations (values are templated) | `{}` | +| `ingress.labels` | Custom labels | `{}` | +| `ingress.path` | Ingress accepted path | `/` | +| `ingress.pathType` | Ingress type of path | `Prefix` | +| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.6/guide/ingress/annotations/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| `ingress.ingressClassName` | Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 | `""` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `extraInitContainers` | Init containers to add to the grafana pod | `{}` | +| `extraContainers` | Sidecar containers to add to the grafana pod | `""` | +| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` | +| `extraLabels` | Custom labels for all manifests | `{}` | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` | +| `persistence.size` | Size of persistent volume claim | `10Gi` | +| `persistence.existingClaim` | Use an existing PVC to persist data (can be templated) | `nil` | +| `persistence.storageClassName` | Type of persistent volume claim | `nil` | +| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` | +| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | +| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | +| `persistence.extraPvcLabels` | Extra labels to apply to a PVC. | `{}` | +| `persistence.subPath` | Mount a sub dir of the persistent volume (can be templated) | `nil` | +| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` | +| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` | +| `initChownData.enabled` | If false, don't reset data ownership at startup | true | +| `initChownData.image.registry` | init-chown-data container image registry | `docker.io` | +| `initChownData.image.repository` | init-chown-data container image repository | `busybox` | +| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | +| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` | +| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` | +| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` | +| `schedulerName` | Alternate scheduler name | `nil` | +| `env` | Extra environment variables passed to pods | `{}` | +| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `envFromSecrets` | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envFromConfigMaps` | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret. (passed through [tpl](https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function)) | `{}` | +| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` | +| `extraSecretMounts` | Additional grafana server secret mounts | `[]` | +| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | +| `extraVolumes` | Additional Grafana server volumes | `[]` | +| `automountServiceAccountToken` | Mounted the service account token on the grafana pod. Mandatory, if sidecars are enabled | `true` | +| `createConfigmap` | Enable creating the grafana configmap | `true` | +| `extraConfigmapMounts` | Additional grafana server configMap volume mounts (values are templated) | `[]` | +| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | +| `plugins` | Plugins to be loaded along with Grafana | `[]` | +| `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `alerting` | Configure grafana alerting (passed through tpl) | `{}` | +| `notifiers` | Configure grafana notifiers | `{}` | +| `dashboardProviders` | Configure grafana dashboard providers | `{}` | +| `dashboards` | Dashboards to import | `{}` | +| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` | +| `grafana.ini` | Grafana's primary configuration | `{}` | +| `global.imageRegistry` | Global image pull registry for all images. | `null` | +| `global.imagePullSecrets` | Global image pull secrets (can be templated). Allows either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). | `[]` | +| `ldap.enabled` | Enable LDAP authentication | `false` | +| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` | +| `ldap.config` | Grafana's LDAP configuration | `""` | +| `annotations` | Deployment annotations | `{}` | +| `labels` | Deployment labels | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | `{}` | +| `podPortName` | Name of the grafana port on the pod | `grafana` | +| `lifecycleHooks` | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers) | `{}` | +| `sidecar.image.registry` | Sidecar image registry | `quay.io` | +| `sidecar.image.repository` | Sidecar image repository | `kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `1.26.0` | +| `sidecar.image.sha` | Sidecar image sha (optional) | `""` | +| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | +| `sidecar.resources` | Sidecar resources | `{}` | +| `sidecar.securityContext` | Sidecar securityContext | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` | +| `sidecar.alerts.enabled` | Enables the cluster wide search for alerts and adds/updates/deletes them in grafana |`false` | +| `sidecar.alerts.label` | Label that config maps with alerts should have to be added | `grafana_alert` | +| `sidecar.alerts.labelValue` | Label value that config maps with alerts should have to be added | `""` | +| `sidecar.alerts.searchNamespace` | Namespaces list. If specified, the sidecar will search for alerts config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.alerts.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.alerts.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.alerts.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/alerting/reload"` | +| `sidecar.alerts.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.alerts.initAlerts` | Set to true to deploy the alerts sidecar as an initContainer. This is needed if skipReload is true, to load any alerts defined at startup time. | `false` | +| `sidecar.alerts.extraMounts` | Additional alerts sidecar volume mounts. | `[]` | +| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | +| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | +| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | +| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` | +| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` | +| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` | +| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` | +| `sidecar.dashboards.provider.type` | Provider type | `file` | +| `sidecar.dashboards.provider.foldersFromFilesStructure` | Allow Grafana to replicate dashboard structure from filesystem. | `false` | +| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | +| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `""` | +| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | +| `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | +| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | +| `sidecar.dashboards.searchNamespace` | Namespaces list. If specified, the sidecar will search for dashboards config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` | +| `sidecar.dashboards.reloadURL` | Full url of dashboards configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/dashboards/reload"` | +| `sidecar.dashboards.skipReload` | Enabling this omits defining the REQ_USERNAME, REQ_PASSWORD, REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` | +| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | +| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `""` | +| `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.datasources.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` | +| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.datasources.initDatasources` | Set to true to deploy the datasource sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any datasources defined at startup time. | `false` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | +| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | +| `sidecar.notifiers.labelValue` | Label value that config maps with notifiers should have to be added | `""` | +| `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.notifiers.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.notifiers.reloadURL` | Full url of notifier configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/notifications/reload"` | +| `sidecar.notifiers.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.notifiers.initNotifiers` | Set to true to deploy the notifier sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any notifiers defined at startup time. | `false` | +| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | +| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | +| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | +| `admin.existingSecret` | The name of an existing secret containing the admin credentials (can be templated). | `""` | +| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` | +| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` | +| `serviceAccount.automountServiceAccountToken` | Automount the service account token on all pods where is service account is used | `false` | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `serviceAccount.create` | Create service account | `true` | +| `serviceAccount.labels` | ServiceAccount labels | `{}` | +| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` | +| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` | +| `rbac.create` | Create and use RBAC resources | `true` | +| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` | +| `rbac.useExistingRole` | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` | +| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `false` | +| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `false` | +| `rbac.extraRoleRules` | Additional rules to add to the Role | [] | +| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | +| `command` | Define command to be executed by grafana container at startup | `nil` | +| `args` | Define additional args if command is used | `nil` | +| `testFramework.enabled` | Whether to create test-related resources | `true` | +| `testFramework.image.registry` | `test-framework` image registry. | `docker.io` | +| `testFramework.image.repository` | `test-framework` image repository. | `bats/bats` | +| `testFramework.image.tag` | `test-framework` image tag. | `v1.4.1` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.securityContext` | `test-framework` securityContext | `{}` | +| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | +| `downloadDashboards.envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | +| `downloadDashboardsImage.registry` | Curl docker image registry | `docker.io` | +| `downloadDashboardsImage.repository` | Curl docker image repository | `curlimages/curl` | +| `downloadDashboardsImage.tag` | Curl docker image tag | `7.73.0` | +| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` | +| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` | +| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | +| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | +| `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` | +| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` | +| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | +| `serviceMonitor.relabelings` | RelabelConfigs to apply to samples before scraping. | `[]` | +| `serviceMonitor.metricRelabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | +| `revisionHistoryLimit` | Number of old ReplicaSets to retain | `10` | +| `imageRenderer.enabled` | Enable the image-renderer deployment & service | `false` | +| `imageRenderer.image.registry` | image-renderer Image registry | `docker.io` | +| `imageRenderer.image.repository` | image-renderer Image repository | `grafana/grafana-image-renderer` | +| `imageRenderer.image.tag` | image-renderer Image tag | `latest` | +| `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` | +| `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` | +| `imageRenderer.env` | extra env-vars for image-renderer | `{}` | +| `imageRenderer.envValueFrom` | Environment variables for image-renderer from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` | +| `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | +| `imageRenderer.podAnnotations ` | image-renderer image-renderer pod annotation | `{}` | +| `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | +| `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | +| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | +| `imageRenderer.service.portName` | image-renderer service port name | `http` | +| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` | +| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` | +| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` | +| `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | +| `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | +| `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | +| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | +| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | +| `imageRenderer.resources` | Set resource limits for image-renderer pods | `{}` | +| `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` | +| `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` | +| `imageRenderer.affinity` | Affinity settings for pod assignment | `{}` | +| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | +| `networkPolicy.ingress` | Enable the creation of an ingress network policy | `true` | +| `networkPolicy.egress.enabled` | Enable the creation of an egress network policy | `false` | +| `networkPolicy.egress.ports` | An array of ports to allow for the egress | `[]` | +| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` | + +### Example ingress with path + +With grafana 6.3 and above + +```yaml +grafana.ini: + server: + domain: monitoring.example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true +ingress: + enabled: true + hosts: + - "monitoring.example.com" + path: "/grafana" +``` + +### Example of extraVolumeMounts and extraVolumes + +Configure additional volumes with `extraVolumes` and volume mounts with `extraVolumeMounts`. + +Example for `extraVolumeMounts` and corresponding `extraVolumes`: + +```yaml +extraVolumeMounts: + - name: plugins + mountPath: /var/lib/grafana/plugins + subPath: configs/grafana/plugins + readOnly: false + - name: dashboards + mountPath: /var/lib/grafana/dashboards + hostPath: /usr/shared/grafana/dashboards + readOnly: false + +extraVolumes: + - name: plugins + existingClaim: existing-grafana-claim + - name: dashboards + hostPath: /usr/shared/grafana/dashboards +``` + +Volumes default to `emptyDir`. Set to `persistentVolumeClaim`, +`hostPath`, `csi`, or `configMap` for other types. For a +`persistentVolumeClaim`, specify an existing claim name with +`existingClaim`. + +## Import dashboards + +There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +dashboards: + default: + some-dashboard: + json: | + { + "annotations": + + ... + # Complete json file here + ... + + "title": "Some Dashboard", + "uid": "abcd1234", + "version": 1 + } + custom-dashboard: + # This is a path to a file inside the dashboards directory inside the chart directory + file: dashboards/custom-dashboard.json + prometheus-stats: + # Ref: https://grafana.com/dashboards/2 + gnetId: 2 + revision: 2 + datasource: Prometheus + loki-dashboard-quick-search: + gnetId: 12019 + revision: 2 + datasource: + - name: DS_PROMETHEUS + value: Prometheus + - name: DS_LOKI + value: Loki + local-dashboard: + url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json +``` + +## BASE64 dashboards + +Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit) +A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk. +If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk. + +### Gerrit use case + +Gerrit API for download files has the following schema: where {project-name} and +{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard +the url value is + +## Sidecar for dashboards + +If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with +a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported +dashboards are deleted/updated. + +A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside +one configmap is currently not properly mirrored in grafana. + +Example dashboard config: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-dashboard + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + [...] +``` + +## Sidecar for datasources + +If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the data sources in grafana can be imported. + +Should you aim for reloading datasources in Grafana each time the config is changed, set `sidecar.datasources.skipReload: false` and adjust `sidecar.datasources.reloadURL` to `http://..svc.cluster.local/api/admin/provisioning/datasources/reload`. + +Secrets are recommended over configmaps for this usecase because datasources usually contain private +data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example values to add a postgres datasource as a kubernetes secret: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-datasources + labels: + grafana_datasource: 'true' # default value for: sidecar.datasources.label +stringData: + pg-db.yaml: |- + apiVersion: 1 + datasources: + - name: My pg db datasource + type: postgres + url: my-postgresql-db:5432 + user: db-readonly-user + secureJsonData: + password: 'SUperSEcretPa$$word' + jsonData: + database: my_datase + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 1000 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false + # allow users to edit datasources from the UI. + editable: false +``` + +Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): + +```yaml +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + # name of the datasource. Required + - name: Graphite + # datasource type. Required + type: graphite + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: + # basic auth username + basicAuthUser: + # basic auth password + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false +``` + +## Sidecar for notifiers + +If the parameter `sidecar.notifiers.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.notifiers.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the notification channels in grafana can be imported. The secrets must be created before +`helm install` so that the notifiers init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because alert notification channels usually contain +private data like SMTP usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](https://grafana.com/docs/grafana/latest/administration/provisioning/#alert-notification-channels): + +```yaml +notifiers: + - name: notification-channel-1 + type: slack + uid: notifier1 + # either + org_id: 2 + # or + org_name: Main Org. + is_default: true + send_reminder: true + frequency: 1h + disable_resolve_message: false + # See `Supported Settings` section for settings supporter for each + # alert notification type. + settings: + recipient: 'XXX' + token: 'xoxb' + uploadImage: true + url: https://slack.com + +delete_notifiers: + - name: notification-channel-1 + uid: notifier1 + org_id: 2 + - name: notification-channel-2 + # default org_id: 1 +``` + +## Sidecar for alerting resources + +If the parameter `sidecar.alerts.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster (namespace defined by `sidecar.alerts.searchNamespace`) and filters out the ones with +a label as defined in `sidecar.alerts.label` (default is `grafana_alert`). The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported alerting resources are updated, however, deletions are a little more complicated (see below). + +This sidecar can be used to provision alert rules, contact points, notification policies, notification templates and mute timings as shown in [Grafana Documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/). + +To fetch the alert config which will be provisioned, use the alert provisioning API ([Grafana Documentation](https://grafana.com/docs/grafana/next/developers/http_api/alerting_provisioning/)). +You can use either JSON or YAML format. + +Example config for an alert rule: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-alert + labels: + grafana_alert: "1" +data: + k8s-alert.yml: |- + apiVersion: 1 + groups: + - orgId: 1 + name: k8s-alert + [...] +``` + +To delete provisioned alert rules is a two step process, you need to delete the configmap which defined the alert rule +and then create a configuration which deletes the alert rule. + +Example deletion configuration: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: delete-sample-grafana-alert + namespace: monitoring + labels: + grafana_alert: "1" +data: + delete-k8s-alert.yml: |- + apiVersion: 1 + deleteRules: + - orgId: 1 + uid: 16624780-6564-45dc-825c-8bded4ad92d3 +``` + +## Statically provision alerting resources +If you don't need to change alerting resources (alert rules, contact points, notification policies and notification templates) regularly you could use the `alerting` config option instead of the sidecar option above. +This will grab the alerting config and apply it statically at build time for the helm file. + +There are two methods to statically provision alerting configuration in Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +alerting: + team1-alert-rules.yaml: + file: alerting/team1/rules.yaml + team2-alert-rules.yaml: + file: alerting/team2/rules.yaml + team3-alert-rules.yaml: + file: alerting/team3/rules.yaml + notification-policies.yaml: + file: alerting/shared/notification-policies.yaml + notification-templates.yaml: + file: alerting/shared/notification-templates.yaml + contactpoints.yaml: + apiVersion: 1 + contactPoints: + - orgId: 1 + name: Slack channel + receivers: + - uid: default-receiver + type: slack + settings: + # Webhook URL to be filled in + url: "" + # We need to escape double curly braces for the tpl function. + text: '{{ `{{ template "default.message" . }}` }}' + title: '{{ `{{ template "default.title" . }}` }}' +``` + +The two possibilities for static alerting resource provisioning are: + +* Inlining the file contents as shown for contact points in the above example. +* Importing a file using a relative path starting from the chart root directory as shown for the alert rules in the above example. + +### Important notes on file provisioning + +* The format of the files is defined in the [Grafana documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/) on file provisioning. +* The chart supports importing YAML and JSON files. +* The filename must be unique, otherwise one volume mount will overwrite the other. +* In case of inlining, double curly braces that arise from the Grafana configuration format and are not intended as templates for the chart must be escaped. +* The number of total files under `alerting:` is not limited. Each file will end up as a volume mount in the corresponding provisioning folder of the deployed Grafana instance. +* The file size for each import is limited by what the function `.Files.Get` can handle, which suffices for most cases. + +## How to serve Grafana with a path prefix (/grafana) + +In order to serve Grafana with a prefix (e.g., ), add the following to your values.yaml. + +```yaml +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + + path: /grafana/?(.*) + hosts: + - k8s.example.dev + +grafana.ini: + server: + root_url: http://localhost:3000/grafana # this host can be localhost +``` + +## How to securely reference secrets in grafana.ini + +This example uses Grafana [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. + +In grafana.ini: + +```yaml +grafana.ini: + [auth.generic_oauth] + enabled = true + client_id = $__file{/etc/secrets/auth_generic_oauth/client_id} + client_secret = $__file{/etc/secrets/auth_generic_oauth/client_secret} +``` + +Existing secret, or created along with helm: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: auth-generic-oauth-secret +type: Opaque +stringData: + client_id: + client_secret: +``` + +Include in the `extraSecretMounts` configuration flag: + +```yaml +- extraSecretMounts: + - name: auth-generic-oauth-secret-mount + secretName: auth-generic-oauth-secret + defaultMode: 0440 + mountPath: /etc/secrets/auth_generic_oauth + readOnly: true +``` + +### extraSecretMounts using a Container Storage Interface (CSI) provider + +This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) + +```yaml +- extraSecretMounts: + - name: secrets-store-inline + mountPath: /run/secrets + readOnly: true + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "my-provider" + nodePublishSecretRef: + name: akv-creds +``` + +## Image Renderer Plug-In + +This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/README.md#run-in-docker) + +```yaml +imageRenderer: + enabled: true +``` + +### Image Renderer NetworkPolicy + +By default the image-renderer pods will have a network policy which only allows ingress traffic from the created grafana instance + +### High Availability for unified alerting + +If you want to run Grafana in a high availability cluster you need to enable +the headless service by setting `headlessService: true` in your `values.yaml` +file. + +As next step you have to setup the `grafana.ini` in your `values.yaml` in a way +that it will make use of the headless service to obtain all the IPs of the +cluster. You should replace ``{{ Name }}`` with the name of your helm deployment. + +```yaml +grafana.ini: + ... + unified_alerting: + enabled: true + ha_peers: {{ Name }}-headless:9094 + ha_listen_address: ${POD_IP}:9094 + ha_advertise_address: ${POD_IP}:9094 + + alerting: + enabled: false +``` diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/dashboards/custom-dashboard.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/dashboards/custom-dashboard.json new file mode 100644 index 0000000000..9e26dfeeb6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/dashboards/custom-dashboard.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/NOTES.txt new file mode 100644 index 0000000000..d86419fe23 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/NOTES.txt @@ -0,0 +1,55 @@ +1. Get your '{{ .Values.adminUser }}' user password by running: + + kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} -o jsonpath="{.data.{{ .Values.admin.passwordKey | default "admin-password" }}}" | base64 --decode ; echo + + +2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}.svc.cluster.local +{{ if .Values.ingress.enabled }} + If you bind grafana to 80, please update values in values.yaml and reinstall: + ``` + securityContext: + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + + command: + - "setcap" + - "'cap_net_bind_service=+ep'" + - "/usr/sbin/grafana-server &&" + - "sh" + - "/run.sh" + ``` + Details refer to https://grafana.com/docs/installation/configuration/#http-port. + Or grafana would always crash. + + From outside the cluster, the server URL(s) are: + {{- range .Values.ingress.hosts }} + http://{{ . }} + {{- end }} +{{- else }} + Get the Grafana URL to visit by running these commands in the same shell: + {{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "grafana.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT + {{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc --namespace {{ include "grafana.namespace" . }} -w {{ include "grafana.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ include "grafana.namespace" . }} {{ include "grafana.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + http://$SERVICE_IP:{{ .Values.service.port -}} + {{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ include "grafana.namespace" . }} -l "app.kubernetes.io/name={{ include "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ include "grafana.namespace" . }} port-forward $POD_NAME 3000 + {{- end }} +{{- end }} + +3. Login with the password from step 1 and the username: {{ .Values.adminUser }} + +{{- if not .Values.persistence.enabled }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Grafana pod is terminated. ##### +################################################################################# +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_config.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_config.tpl new file mode 100644 index 0000000000..19df19cd2a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_config.tpl @@ -0,0 +1,171 @@ +{{/* + Generate config map data + */}} +{{- define "grafana.configData" -}} +{{ include "grafana.assertNoLeakedSecrets" . }} +{{- $files := .Files }} +{{- $root := . -}} +{{- with .Values.plugins }} +plugins: {{ join "," . }} +{{- end }} +grafana.ini: | +{{- range $elem, $elemVal := index .Values "grafana.ini" }} + {{- if not (kindIs "map" $elemVal) }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} +{{- end }} +{{- range $key, $value := index .Values "grafana.ini" }} + {{- if kindIs "map" $value }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- range $key, $value := .Values.datasources }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.notifiers }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.alerting }} +{{- if (hasKey $value "file") }} +{{ $key }}: +{{- toYaml ( $files.Get $value.file ) | nindent 2 }} +{{- else if (or (hasKey $value "secret") (hasKey $value "secretFile"))}} +{{/* will be stored inside secret generated by "configSecret.yaml"*/}} +{{- else }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.dashboardProviders }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} + +{{- if .Values.dashboards }} +download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} +{{ $dashboardProviders := .Values.dashboardProviders }} +{{- range $provider, $dashboards := .Values.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -skf \ + --connect-timeout 60 \ + --max-time 60 \ + {{- if not $value.b64content }} + {{- if not $value.acceptHeader }} + -H "Accept: application/json" \ + {{- else }} + -H "Accept: {{ $value.acceptHeader }}" \ + {{- end }} + {{- if $value.token }} + -H "Authorization: token {{ $value.token }}" \ + {{- end }} + {{- if $value.bearerToken }} + -H "Authorization: Bearer {{ $value.bearerToken }}" \ + {{- end }} + {{- if $value.basic }} + -H "Authorization: Basic {{ $value.basic }}" \ + {{- end }} + {{- if $value.gitlabToken }} + -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \ + {{- end }} + -H "Content-Type: application/json;charset=UTF-8" \ + {{- end }} + {{- $dpPath := "" -}} + {{- range $kd := (index $dashboardProviders "dashboardproviders.yaml").providers }} + {{- if eq $kd.name $provider }} + {{- $dpPath = $kd.options.path }} + {{- end }} + {{- end }} + {{- if $value.url }} + "{{ $value.url }}" \ + {{- else }} + "https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download" \ + {{- end }} + {{- if $value.datasource }} + {{- if kindIs "string" $value.datasource }} + | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g' \ + {{- end }} + {{- if kindIs "slice" $value.datasource }} + {{- range $value.datasource }} + | sed '/-- .* --/! s/${{"{"}}{{ .name }}}/{{ .value }}/g' \ + {{- end }} + {{- end }} + {{- end }} + {{- if $value.b64content }} + | base64 -d \ + {{- end }} + > "{{- if $dpPath -}}{{ $dpPath }}{{- else -}}/var/lib/grafana/dashboards/{{ $provider }}{{- end -}}/{{ $key }}.json" + {{ end }} + {{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Generate dashboard json config map data + */}} +{{- define "grafana.configDashboardProviderData" -}} +provider.yaml: |- + apiVersion: 1 + providers: + - name: '{{ .Values.sidecar.dashboards.provider.name }}' + orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + {{- end }} + type: {{ .Values.sidecar.dashboards.provider.type }} + disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} + allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }} + options: + foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} +{{- end -}} + +{{- define "grafana.secretsData" -}} +{{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} +admin-user: {{ .Values.adminUser | b64enc | quote }} +{{- if .Values.adminPassword }} +admin-password: {{ .Values.adminPassword | b64enc | quote }} +{{- else }} +admin-password: {{ include "grafana.password" . }} +{{- end }} +{{- end }} +{{- if not .Values.ldap.existingSecret }} +ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_helpers.tpl new file mode 100644 index 0000000000..68d2d815d8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,305 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "grafana.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "grafana.serviceAccountNameTest" -}} +{{- if .Values.serviceAccount.create }} +{{- default (print (include "grafana.fullname" .) "-test") .Values.serviceAccount.nameTest }} +{{- else }} +{{- default "default" .Values.serviceAccount.nameTest }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "grafana.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.extraLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "grafana.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.imageRenderer.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.imageRenderer.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels ImageRenderer +*/}} +{{- define "grafana.imageRenderer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Looks if there's an existing secret and reuse its password. If not it generates +new password and use it. +*/}} +{{- define "grafana.password" -}} +{{- $secret := (lookup "v1" "Secret" (include "grafana.namespace" .) (include "grafana.fullname" .) ) }} +{{- if $secret }} +{{- index $secret "data" "admin-password" }} +{{- else }} +{{- (randAlphaNum 40) | b64enc | quote }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for rbac. +*/}} +{{- define "grafana.rbac.apiVersion" -}} +{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} +{{- print "rbac.authorization.k8s.io/v1" }} +{{- else }} +{{- print "rbac.authorization.k8s.io/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "grafana.ingress.apiVersion" -}} +{{- if and ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) }} +{{- print "networking.k8s.io/v1" }} +{{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +{{- print "networking.k8s.io/v1beta1" }} +{{- else }} +{{- print "extensions/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "grafana.hpa.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "autoscaling/v2" }} +{{- print "autoscaling/v2" }} +{{- else }} +{{- print "autoscaling/v2beta2" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "grafana.podDisruptionBudget.apiVersion" -}} +{{- if $.Values.podDisruptionBudget.apiVersion }} +{{- print $.Values.podDisruptionBudget.apiVersion }} +{{- else if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" }} +{{- print "policy/v1" }} +{{- else }} +{{- print "policy/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return if ingress is stable. +*/}} +{{- define "grafana.ingress.isStable" -}} +{{- eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1" }} +{{- end }} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "grafana.ingress.supportsIngressClassName" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "grafana.ingress.supportsPathType" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "root" . "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "grafana.imagePullSecrets" -}} +{{- $root := .root }} +{{- range (concat .root.Values.global.imagePullSecrets .imagePullSecrets) }} +{{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml (dict "name" (tpl .name $root)) | trim }} +{{- else }} +- name: {{ tpl . $root }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* + Checks whether or not the configSecret secret has to be created + */}} +{{- define "grafana.shouldCreateConfigSecret" -}} +{{- $secretFound := false -}} +{{- range $key, $value := .Values.datasources }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} + {{- if (or (hasKey $value "secret") (hasKey $value "secretFile")) }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- $secretFound}} +{{- end -}} + +{{/* + Checks whether the user is attempting to store secrets in plaintext + in the grafana.ini configmap +*/}} +{{/* grafana.assertNoLeakedSecrets checks for sensitive keys in values */}} +{{- define "grafana.assertNoLeakedSecrets" -}} + {{- $sensitiveKeysYaml := ` +sensitiveKeys: +- path: ["database", "password"] +- path: ["smtp", "password"] +- path: ["security", "secret_key"] +- path: ["security", "admin_password"] +- path: ["auth.basic", "password"] +- path: ["auth.ldap", "bind_password"] +- path: ["auth.google", "client_secret"] +- path: ["auth.github", "client_secret"] +- path: ["auth.gitlab", "client_secret"] +- path: ["auth.generic_oauth", "client_secret"] +- path: ["auth.okta", "client_secret"] +- path: ["auth.azuread", "client_secret"] +- path: ["auth.grafana_com", "client_secret"] +- path: ["auth.grafananet", "client_secret"] +- path: ["azure", "user_identity_client_secret"] +- path: ["unified_alerting", "ha_redis_password"] +- path: ["metrics", "basic_auth_password"] +- path: ["external_image_storage.s3", "secret_key"] +- path: ["external_image_storage.webdav", "password"] +- path: ["external_image_storage.azure_blob", "account_key"] +` | fromYaml -}} + {{- if $.Values.assertNoLeakedSecrets -}} + {{- $grafanaIni := index .Values "grafana.ini" -}} + {{- range $_, $secret := $sensitiveKeysYaml.sensitiveKeys -}} + {{- $currentMap := $grafanaIni -}} + {{- $shouldContinue := true -}} + {{- range $index, $elem := $secret.path -}} + {{- if and $shouldContinue (hasKey $currentMap $elem) -}} + {{- if eq (len $secret.path) (add1 $index) -}} + {{- if not (regexMatch "\\$(?:__(?:env|file|vault))?{[^}]+}" (index $currentMap $elem)) -}} + {{- fail (printf "Sensitive key '%s' should not be defined explicitly in values. Use variable expansion instead. You can disable this client-side validation by changing the value of assertNoLeakedSecrets." (join "." $secret.path)) -}} + {{- end -}} + {{- else -}} + {{- $currentMap = index $currentMap $elem -}} + {{- end -}} + {{- else -}} + {{- $shouldContinue = false -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_pod.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_pod.tpl new file mode 100644 index 0000000000..2ebf7d5f10 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_pod.tpl @@ -0,0 +1,1296 @@ +{{- define "grafana.pod" -}} +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- $root := . -}} +{{- with .Values.schedulerName }} +schedulerName: "{{ . }}" +{{- end }} +serviceAccountName: {{ include "grafana.serviceAccountName" . }} +automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} +{{- with .Values.securityContext }} +securityContext: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.hostAliases }} +hostAliases: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .Values.dnsPolicy }} +dnsPolicy: {{ .Values.dnsPolicy }} +{{- end }} +{{- with .Values.dnsConfig }} +dnsConfig: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.priorityClassName }} +priorityClassName: {{ . }} +{{- end }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.extraInitContainers (and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts) (and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources) (and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers)) }} +initContainers: +{{- end }} +{{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} + - name: init-chown-data + {{- $registry := include "system_default_registry" . | default .Values.initChownData.image.registry -}} + {{- if .Values.initChownData.image.sha }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}@sha256:{{ .Values.initChownData.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.initChownData.image.pullPolicy }} + {{- with .Values.initChownData.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + command: + - chown + - -R + - {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }} + - /var/lib/grafana + {{- with .Values.initChownData.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} +{{- end }} +{{- if .Values.dashboards }} + - name: download-dashboards + {{- $registry := include "system_default_registry" . | default .Values.downloadDashboardsImage.registry -}} + {{- if .Values.downloadDashboardsImage.sha }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}@sha256:{{ .Values.downloadDashboardsImage.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} + command: ["/bin/sh"] + args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh -x /etc/grafana/download_dashboards.sh" ] + {{- with .Values.downloadDashboards.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + env: + {{- range $key, $value := .Values.downloadDashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.downloadDashboards.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- with .Values.downloadDashboards.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.downloadDashboards.envFromSecret }} + envFrom: + - secretRef: + name: {{ tpl . $root }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts }} + - name: {{ include "grafana.name" . }}-init-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources }} + - name: {{ include "grafana.name" . }}-init-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.datasources.searchNamespace | join ",") . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end }} +{{- if and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers }} + - name: {{ include "grafana.name" . }}-init-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- with .Values.extraInitContainers }} + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 2 }} +{{- end }} +{{- if not .Values.enableKubeBackwardCompatibility }} +enableServiceLinks: {{ .Values.enableServiceLinks }} +{{- end }} +containers: +{{- if and .Values.sidecar.alerts.enabled (not .Values.sidecar.alerts.initAlerts) }} + - name: {{ include "grafana.name" . }}-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.alerts.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.alerts.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.alerts.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.alerts.watchServerTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchServerTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.alerts.watchClientTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchClientTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if .Values.sidecar.dashboards.enabled }} + - name: {{ include "grafana.name" . }}-sc-dashboard + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.dashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.datasources.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.dashboards.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.dashboards.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.dashboards.label }}" + {{- with .Values.sidecar.dashboards.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.dashboards.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.folderAnnotation }} + - name: FOLDER_ANNOTATION + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- if not .Values.sidecar.dashboards.skipReload }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + - name: REQ_URL + value: {{ .Values.sidecar.dashboards.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.dashboards.watchServerTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchServerTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchClientTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchClientTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: {{ .Values.sidecar.dashboards.watchClientTimeout | quote }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- with .Values.sidecar.dashboards.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if and .Values.sidecar.datasources.enabled (not .Values.sidecar.datasources.initDatasources) }} + - name: {{ include "grafana.name" . }}-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.datasources.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.datasources.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.datasources.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.datasources.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.datasources.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.datasources.watchServerTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchServerTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.watchClientTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchClientTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: {{ include "grafana.name" . }}-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.notifiers.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.notifiers.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.notifiers.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.notifiers.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.notifiers.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.notifiers.watchServerTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchServerTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.notifiers.watchClientTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchClientTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: {{ include "grafana.name" . }}-sc-plugins + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.plugins.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.plugins.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.plugins.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.plugins.label }}" + {{- if .Values.sidecar.plugins.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.plugins.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/plugins" + - name: RESOURCE + value: {{ quote .Values.sidecar.plugins.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.plugins.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.plugins.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.plugins.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.plugins.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.plugins.watchServerTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchServerTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.watchClientTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchClientTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" +{{- end}} + - name: {{ .Chart.Name }} + {{- $registry := include "system_default_registry" . | default .Values.image.registry -}} + {{- if .Values.image.sha }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .Values.args }} + args: + {{- range .Values.args }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + {{- if .Values.ldap.enabled }} + - name: ldap + mountPath: "/etc/grafana/ldap.toml" + subPath: ldap.toml + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ tpl (.subPath | default "") $root }} + readOnly: {{ .readOnly }} + {{- end }} + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- with .Values.dashboards }} + {{- range $provider, $dashboards := . }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "json") (hasKey $value "file")) }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardsConfigMaps }} + {{- range (keys . | sortAlpha) }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" + {{- end }} + {{- end }} + {{- with .Values.datasources }} + {{- $datasources := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $datasources .) "secret")) }} {{/*check if current datasource should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.notifiers }} + {{- $notifiers := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $notifiers .) "secret")) }} {{/*check if current notifier should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.alerting }} + {{- $alertingmap := .}} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $.Values.alerting .) "secret") (hasKey (index $.Values.alerting .) "secretFile")) }} {{/*check if current alerting entry should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardProviders }} + {{- range (keys . | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- end}} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml + {{- end}} + {{- end}} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" + {{- end}} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" + {{- end}} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" + {{- end}} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + ports: + - name: {{ .Values.podPortName }} + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + - name: {{ .Values.gossipPortName }}-tcp + containerPort: 9094 + protocol: TCP + - name: {{ .Values.gossipPortName }}-udp + containerPort: 9094 + protocol: UDP + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if .Values.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ include "grafana.fullname" . }} + key: plugins + {{- end }} + {{- if .Values.smtp.existingSecret }} + - name: GF_SMTP_USER + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.userKey | default "user" }} + - name: GF_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.passwordKey | default "password" }} + {{- end }} + {{- if .Values.imageRenderer.enabled }} + - name: GF_RENDERING_SERVER_URL + value: http://{{ include "grafana.fullname" . }}-image-renderer.{{ include "grafana.namespace" . }}:{{ .Values.imageRenderer.service.port }}/render + - name: GF_RENDERING_CALLBACK_URL + value: {{ .Values.imageRenderer.grafanaProtocol }}://{{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} + {{- end }} + - name: GF_PATHS_DATA + value: {{ (get .Values "grafana.ini").paths.data }} + - name: GF_PATHS_LOGS + value: {{ (get .Values "grafana.ini").paths.logs }} + - name: GF_PATHS_PLUGINS + value: {{ (get .Values "grafana.ini").paths.plugins }} + - name: GF_PATHS_PROVISIONING + value: {{ (get .Values "grafana.ini").paths.provisioning }} + {{- range $key, $value := .Values.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- range $key, $value := .Values.env }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" + {{- end }} + {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) .Values.envFromConfigMaps }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ tpl .Values.envFromSecret . }} + {{- end }} + {{- if .Values.envRenderSecret }} + - secretRef: + name: {{ include "grafana.fullname" . }}-env + {{- end }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- range .Values.envFromConfigMaps }} + - configMapRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.lifecycleHooks }} + lifecycle: + {{- tpl (toYaml .) $root | nindent 6 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- with .Values.extraContainers }} + {{- tpl . $ | nindent 2 }} +{{- end }} +nodeSelector: {{ include "linux-node-selector" . | nindent 2 }} +{{- with .Values.nodeSelector }} + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.affinity }} +affinity: + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- with .Values.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} +tolerations: {{ include "linux-node-tolerations" . | nindent 2 }} +{{- with .Values.tolerations }} + {{- toYaml . | nindent 2 }} +{{- end }} +volumes: + - name: config + configMap: + name: {{ include "grafana.fullname" . }} + {{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} + {{- if and .Values.createConfigmap $createConfigSecret }} + - name: config-secret + secret: + secretName: {{ include "grafana.fullname" . }}-config-secret + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.dashboards }} + {{- range (keys .Values.dashboards | sortAlpha) }} + - name: dashboards-{{ . }} + configMap: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.dashboardsConfigMaps }} + {{- range $provider, $name := .Values.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ tpl $name $root }} + {{- end }} + {{- end }} + {{- if .Values.ldap.enabled }} + - name: ldap + secret: + {{- if .Values.ldap.existingSecret }} + secretName: {{ .Values.ldap.existingSecret }} + {{- else }} + secretName: {{ include "grafana.fullname" . }} + {{- end }} + items: + - key: ldap-toml + path: ldap.toml + {{- end }} + {{- if and .Values.persistence.enabled (eq .Values.persistence.type "pvc") }} + - name: storage + persistentVolumeClaim: + claimName: {{ tpl (.Values.persistence.existingClaim | default (include "grafana.fullname" .)) . }} + {{- else if and .Values.persistence.enabled (has .Values.persistence.type $sts) }} + {{/* nothing */}} + {{- else }} + - name: storage + {{- if .Values.persistence.inMemory.enabled }} + emptyDir: + medium: Memory + {{- with .Values.persistence.inMemory.sizeLimit }} + sizeLimit: {{ . }} + {{- end }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + emptyDir: + {{- with .Values.sidecar.alerts.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + emptyDir: + {{- with .Values.sidecar.dashboards.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + configMap: + name: {{ include "grafana.fullname" . }}-config-dashboards + {{- end }} + {{- end }} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + emptyDir: + {{- with .Values.sidecar.datasources.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + emptyDir: + {{- with .Values.sidecar.plugins.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + emptyDir: + {{- with .Values.sidecar.notifiers.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- range .Values.extraSecretMounts }} + {{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else if .projected }} + - name: {{ .name }} + projected: + {{- toYaml .projected | nindent 6 }} + {{- else if .csi }} + - name: {{ .name }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- end }} + {{- end }} + {{- range .Values.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 6 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 6 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 6 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + emptyDir: {} + {{- end }} + {{- with .Values.extraContainerVolumes }} + {{- tpl (toYaml .) $root | nindent 2 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrole.yaml new file mode 100644 index 0000000000..3af4b62b63 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) (not .Values.rbac.useExistingClusterRole) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-clusterrole +{{- if or .Values.sidecar.dashboards.enabled .Values.rbac.extraClusterRoleRules .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} +rules: + {{- if or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end}} + {{- with .Values.rbac.extraClusterRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end}} +{{- end}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..bda9431a2c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "grafana.fullname" . }}-clusterrolebinding + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +roleRef: + kind: ClusterRole + {{- if .Values.rbac.useExistingClusterRole }} + name: {{ .Values.rbac.useExistingClusterRole }} + {{- else }} + name: {{ include "grafana.fullname" . }}-clusterrole + {{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configSecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configSecret.yaml new file mode 100644 index 0000000000..55574b9bbc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configSecret.yaml @@ -0,0 +1,43 @@ +{{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} +{{- if and .Values.createConfigmap $createConfigSecret }} +{{- $files := .Files }} +{{- $root := . -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ include "grafana.fullname" . }}-config-secret" + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- range $key, $value := .Values.alerting }} + {{- if (hasKey $value "secretFile") }} + {{- $key | nindent 2 }}: + {{- toYaml ( $files.Get $value.secretFile ) | b64enc | nindent 4}} + {{/* as of https://helm.sh/docs/chart_template_guide/accessing_files/ this will only work if you fork this chart and add files to it*/}} + {{- end }} +{{- end }} +stringData: +{{- range $key, $value := .Values.datasources }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} +{{ if (hasKey $value "secret") }} + {{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml new file mode 100644 index 0000000000..b412c4d1f0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.sidecar.dashboards.enabled .Values.sidecar.dashboards.SCProvider }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-config-dashboards + namespace: {{ include "grafana.namespace" . }} +data: + {{- include "grafana.configDashboardProviderData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap.yaml new file mode 100644 index 0000000000..7d7428be51 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap.yaml @@ -0,0 +1,15 @@ +{{- if .Values.createConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- include "grafana.configData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml new file mode 100644 index 0000000000..b96ce72026 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml @@ -0,0 +1,38 @@ +{{- if .Values.dashboards }} +{{ $files := .Files }} +{{- range $provider, $dashboards := .Values.dashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ include "grafana.namespace" $ }} + labels: + {{- include "grafana.labels" $ | nindent 4 }} + dashboard-provider: {{ $provider }} + {{- if $.Values.sidecar.dashboards.enabled }} + {{ $.Values.sidecar.dashboards.label }}: {{ $.Values.sidecar.dashboards.labelValue | quote }} + {{- end }} +{{- if $dashboards }} +data: +{{- $dashboardFound := false }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} +{{- $dashboardFound = true }} + {{- print $key | nindent 2 }}.json: + {{- if hasKey $value "json" }} + |- + {{- $value.json | nindent 6 }} + {{- end }} + {{- if hasKey $value "file" }} + {{- toYaml ( $files.Get $value.file ) | nindent 4}} + {{- end }} +{{- end }} +{{- end }} +{{- if not $dashboardFound }} + {} +{{- end }} +{{- end }} +--- +{{- end }} + +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/deployment.yaml new file mode 100644 index 0000000000..46c016faa3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/deployment.yaml @@ -0,0 +1,53 @@ +{{- if (and (not .Values.useStatefulSet) (or (not .Values.persistence.enabled) (eq .Values.persistence.type "pvc"))) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }} + replicas: {{ .Values.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + {{- with .Values.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include "grafana.configData" . | sha256sum }} + {{- if .Values.dashboards }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + {{- end }} + checksum/sc-dashboard-provider-config: {{ include "grafana.configDashboardProviderData" . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include "grafana.secretsData" . | sha256sum }} + {{- end }} + {{- if .Values.envRenderSecret }} + checksum/secret-env: {{ tpl (toYaml .Values.envRenderSecret) . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/extra-manifests.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/extra-manifests.yaml new file mode 100644 index 0000000000..a9bb3b6ba8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraObjects }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/headless-service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/headless-service.yaml new file mode 100644 index 0000000000..3028589d32 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/headless-service.yaml @@ -0,0 +1,22 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if or .Values.headlessService (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-headless + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + clusterIP: None + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + type: ClusterIP + ports: + - name: {{ .Values.gossipPortName }}-tcp + port: 9094 +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/hpa.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/hpa.yaml new file mode 100644 index 0000000000..46bbcb49a2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/hpa.yaml @@ -0,0 +1,52 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if .Values.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }} + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + {{- if has .Values.persistence.type $sts }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ include "grafana.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.behavior }} + behavior: {{ toYaml .Values.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml new file mode 100644 index 0000000000..28231b803e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml @@ -0,0 +1,131 @@ +{{ if .Values.imageRenderer.enabled }} +{{- $root := . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.imageRenderer.autoscaling.enabled) (.Values.imageRenderer.replicas) }} + replicas: {{ .Values.imageRenderer.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + + {{- with .Values.imageRenderer.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 8 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.imageRenderer.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imageRenderer.schedulerName }} + schedulerName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.serviceAccountName }} + serviceAccountName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.image.pullSecrets }} + imagePullSecrets: + {{- range . }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + containers: + - name: {{ .Chart.Name }}-image-renderer + {{- $registry := include "system_default_registry" | default .Values.imageRenderer.image.registry -}} + {{- if .Values.imageRenderer.image.sha }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}@sha256:{{ .Values.imageRenderer.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.imageRenderer.image.pullPolicy }} + {{- if .Values.imageRenderer.command }} + command: + {{- range .Values.imageRenderer.command }} + - {{ . }} + {{- end }} + {{- end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + containerPort: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.imageRenderer.service.portName }} + env: + - name: HTTP_PORT + value: {{ .Values.imageRenderer.service.targetPort | quote }} + {{- if .Values.imageRenderer.serviceMonitor.enabled }} + - name: ENABLE_METRICS + value: "true" + {{- end }} + {{- range $key, $value := .Values.imageRenderer.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 16 }} + {{- end }} + {{- range $key, $value := .Values.imageRenderer.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- with .Values.imageRenderer.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /tmp + name: image-renderer-tmpfs + {{- with .Values.imageRenderer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.imageRenderer.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: image-renderer-tmpfs + emptyDir: {} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml new file mode 100644 index 0000000000..b0f0059b79 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "grafana.fullname" . }}-image-renderer + minReplicas: {{ .Values.imageRenderer.autoscaling.minReplicas }} + maxReplicas: {{ .Values.imageRenderer.autoscaling.maxReplicas }} + metrics: + {{- if .Values.imageRenderer.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.behavior }} + behavior: {{ toYaml .Values.imageRenderer.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml new file mode 100644 index 0000000000..d1a0eb313d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitIngress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-ingress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer ingress traffic from grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Ingress + ingress: + - ports: + - port: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.imageRenderer.networkPolicy.extraIngressSelectors -}} + {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} + +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitEgress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-egress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer egress traffic to grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Egress + egress: + # allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # talk only to grafana + - ports: + - port: {{ .Values.service.targetPort }} + protocol: TCP + to: + - namespaceSelector: + matchLabels: + name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-service.yaml new file mode 100644 index 0000000000..f8da127cf8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-service.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + {{- with .Values.imageRenderer.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + port: {{ .Values.imageRenderer.service.port }} + protocol: TCP + targetPort: {{ .Values.imageRenderer.service.targetPort }} + {{- with .Values.imageRenderer.appProtocol }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml new file mode 100644 index 0000000000..5d9f09d266 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if .Values.imageRenderer.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + {{- if .Values.imageRenderer.serviceMonitor.namespace }} + namespace: {{ tpl .Values.imageRenderer.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.imageRenderer.service.portName }} + {{- with .Values.imageRenderer.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.imageRenderer.serviceMonitor.path }} + scheme: {{ .Values.imageRenderer.serviceMonitor.scheme }} + {{- with .Values.imageRenderer.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}-image-renderer" + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.imageRenderer.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/ingress.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/ingress.yaml new file mode 100644 index 0000000000..b2ffd81095 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/ingress.yaml @@ -0,0 +1,78 @@ +{{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "grafana.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "grafana.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "grafana.ingress.supportsPathType" .) "true" -}} +{{- $fullName := include "grafana.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +{{- $extraPaths := .Values.ingress.extraPaths -}} +apiVersion: {{ include "grafana.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} + {{- with .Values.ingress.tls }} + tls: + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} + rules: + {{- if .Values.ingress.hosts }} + {{- range .Values.ingress.hosts }} + - host: {{ tpl . $ | quote }} + http: + paths: + {{- with $extraPaths }} + {{- toYaml . | nindent 10 }} + {{- end }} + - path: {{ $ingressPath }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end }} + {{- else }} + - http: + paths: + - backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- with $ingressPath }} + path: {{ . }} + {{- end }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/networkpolicy.yaml new file mode 100644 index 0000000000..4cd3ed6976 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/networkpolicy.yaml @@ -0,0 +1,61 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + policyTypes: + {{- if .Values.networkPolicy.ingress }} + - Ingress + {{- end }} + {{- if .Values.networkPolicy.egress.enabled }} + - Egress + {{- end }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + + {{- if .Values.networkPolicy.egress.enabled }} + egress: + {{- if not .Values.networkPolicy.egress.blockDNSResolution }} + - ports: + - port: 53 + protocol: UDP + {{- end }} + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- with .Values.networkPolicy.egress.to }} + to: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.networkPolicy.ingress }} + ingress: + - ports: + - port: {{ .Values.service.targetPort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ include "grafana.fullname" . }}-client: "true" + {{- with .Values.networkPolicy.explicitNamespacesSelector }} + - namespaceSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "grafana.labels" . | nindent 14 }} + role: read + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/nginx-config.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/nginx-config.yaml new file mode 100644 index 0000000000..557471f6ff --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/nginx-config.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-nginx-proxy-config + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 8080; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location /api/dashboards { + proxy_pass http://localhost:3000; + } + + location /api/search { + proxy_pass http://localhost:3000; + + sub_filter_types application/json; + sub_filter_once off; + } + + location /api/live/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $http_host; + proxy_pass http://localhost:3000; + } + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:3000/; + + sub_filter_once off; + + {{- if eq .Values.global.cattle.clusterId "local" -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- else -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- end -}} + + sub_filter ':"/avatar/' ':"avatar/'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000000..05251214ac --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: {{ include "grafana.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..973caccd57 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml @@ -0,0 +1,45 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + # Default set from Docker, with DAC_OVERRIDE and CHOWN + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'csi' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/pvc.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/pvc.yaml new file mode 100644 index 0000000000..c9b234305f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/pvc.yaml @@ -0,0 +1,41 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.persistence.extraPvcLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.finalizers }} + finalizers: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)) }} + volumeName: {{ (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)).spec.volumeName }} + {{- end }} + {{- with .Values.persistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/role.yaml new file mode 100644 index 0000000000..469b6f4e6c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/role.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)) }} +rules: + {{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}] + {{- end }} + {{- if and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled) }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- with .Values.rbac.extraRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/rolebinding.yaml new file mode 100644 index 0000000000..58f77c6b0b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/rolebinding.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + {{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} + {{- else }} + name: {{ include "grafana.fullname" . }} + {{- end }} +subjects: +- kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret-env.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret-env.yaml new file mode 100644 index 0000000000..eb14aac707 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret-env.yaml @@ -0,0 +1,14 @@ +{{- if .Values.envRenderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }}-env + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $key, $val := .Values.envRenderSecret }} + {{ $key }}: {{ tpl ($val | toString) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret.yaml new file mode 100644 index 0000000000..fd2ca50f4b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret.yaml @@ -0,0 +1,16 @@ +{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- include "grafana.secretsData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/service.yaml new file mode 100644 index 0000000000..e9396a15c6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/service.yaml @@ -0,0 +1,61 @@ +{{- if .Values.service.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} +spec: + {{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- with .Values.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- else if eq .Values.service.type "LoadBalancer" }} + type: LoadBalancer + {{- with .Values.service.loadBalancerIP }} + loadBalancerIP: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerClass }} + loadBalancerClass: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} + type: {{ .Values.service.type }} + {{- end }} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} + {{- with .Values.service.appProtocol }} + appProtocol: {{ . }} + {{- end }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- with .Values.extraExposePorts }} + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/serviceaccount.yaml new file mode 100644 index 0000000000..ffca0717ae --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.serviceAccount.autoMount | default .Values.serviceAccount.automountServiceAccountToken }} +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/servicemonitor.yaml new file mode 100644 index 0000000000..b321b1269c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/servicemonitor.yaml @@ -0,0 +1,68 @@ +{{- if .Values.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ tpl .Values.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- with .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + metricRelabelings: + {{- if .Values.serviceMonitor.metricRelabelings }} + {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.serviceMonitor.relabelings }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + {{- with .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}" + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/statefulset.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/statefulset.yaml new file mode 100644 index 0000000000..49278083e8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/statefulset.yaml @@ -0,0 +1,58 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if (or (.Values.useStatefulSet) (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)))}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + serviceName: {{ include "grafana.fullname" . }}-headless + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} + {{- if .Values.persistence.enabled}} + volumeClaimTemplates: + - metadata: + name: storage + spec: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + accessModes: {{ .Values.persistence.accessModes }} + storageClassName: {{ .Values.persistence.storageClassName }} + resources: + requests: + storage: {{ required "Must provide size for persistent volumes used by Grafana" .Values.persistence.size }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml new file mode 100644 index 0000000000..01c96c9243 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + run.sh: |- + @test "Test Health" { + url="http://{{ include "grafana.fullname" . }}/api/health" + + code=$(wget --server-response --spider --timeout 90 --tries 10 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml new file mode 100644 index 0000000000..70a0a884c9 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -0,0 +1,32 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }}-test + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +spec: + allowPrivilegeEscalation: true + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + fsGroup: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - projected + - csi + - secret +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-role.yaml new file mode 100644 index 0000000000..976418b137 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-role.yaml @@ -0,0 +1,17 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}-test] +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml new file mode 100644 index 0000000000..509566eccd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "grafana.fullname" . }}-test +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml new file mode 100644 index 0000000000..38fba3596a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.testFramework.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test.yaml new file mode 100644 index 0000000000..83aaa185c2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test.yaml @@ -0,0 +1,53 @@ +{{- if .Values.testFramework.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + namespace: {{ include "grafana.namespace" . }} +spec: + serviceAccountName: {{ include "grafana.serviceAccountNameTest" . }} + {{- with .Values.testFramework.securityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 4 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 4 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: {{ .Release.Name }}-test + image: "{{ template "system_default_registry" . | default .Values.testFramework.image.registry }}/{{ .Values.testFramework.image.repository }}:{{ .Values.testFramework.image.tag }}" + imagePullPolicy: "{{ .Values.testFramework.imagePullPolicy}}" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + {{- with .Values.testFramework.resources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tests + configMap: + name: {{ include "grafana.fullname" . }}-test + restartPolicy: Never +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/values.yaml new file mode 100644 index 0000000000..2bb62aec39 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/values.yaml @@ -0,0 +1,1315 @@ +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # Can be tempalted. + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + +rbac: + create: true + ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true) + # useExistingRole: name-of-some-role + # useExistingClusterRole: name-of-some-clusterRole + pspEnabled: false + pspUseAppArmor: false + namespaced: false + extraRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] + extraClusterRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] +serviceAccount: + create: true + name: + nameTest: + ## ServiceAccount labels. + labels: {} + ## Service account annotations. Can be templated. + # annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::123456789000:role/iam-role-name-here + + ## autoMount is deprecated in favor of automountServiceAccountToken + # autoMount: false + automountServiceAccountToken: true + +replicas: 1 + +## Create a headless service for the deployment +headlessService: false + +## Should the service account be auto mounted on the pod +automountServiceAccountToken: true + +## Create HorizontalPodAutoscaler object for deployment type +# +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + +## See `kubectl explain poddisruptionbudget.spec` for more +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} +# apiVersion: "" +# minAvailable: 1 +# maxUnavailable: 1 + +## See `kubectl explain deployment.spec.strategy` for more +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: RollingUpdate + +readinessProbe: + httpGet: + path: /api/health + port: 3000 + +livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: "default-scheduler" + +image: + repository: rancher/mirrored-grafana-grafana + # Overrides the Grafana image tag whose default is the chart appVersion + tag: 10.3.3 + sha: "" + pullPolicy: IfNotPresent + + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Can be templated. + ## + pullSecrets: [] + # - myRegistrKeySecretName + +testFramework: + enabled: false + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# dns configuration for pod +dnsPolicy: ~ +dnsConfig: {} + # nameservers: + # - 8.8.8.8 + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# Enable creating the grafana configmap +createConfigmap: true + +# Extra configmaps to mount in grafana pods +# Values are templated. +extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + + +extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + +# Apply extra labels to common labels. +extraLabels: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +downloadDashboardsImage: + repository: rancher/mirrored-curlimages-curl + tag: 7.85.0 + sha: "" + pullPolicy: IfNotPresent + +downloadDashboards: + env: {} + envFromSecret: "" + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## Pod Annotations +# podAnnotations: {} + +## Pod Labels +# podLabels: {} + +podPortName: grafana +gossipPortName: gossip +## Deployment annotations +# annotations: {} + +## Expose the grafana service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + enabled: true + type: ClusterIP + loadBalancerIP: "" + loadBalancerClass: "" + loadBalancerSourceRanges: [] + port: 80 + targetPort: 3000 + # targetPort: 4181 To be used with a proxy extraContainer + ## Service annotations. Can be templated. + annotations: {} + labels: {} + portName: service + # Adds the appProtocol field to the service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + +serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 30s + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + metricRelabelings: [] + targetLabels: [] + +extraExposePorts: [] + # - name: keycloak + # port: 8080 + # targetPort: 8080 + +# overrides pod.spec.hostAliases in the grafana deployment's pods +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "my.host.com" + +ingress: + enabled: false + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + # Values can be templated + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + path: / + + # pathType is only for k8s >= 1.1= + pathType: Prefix + + hosts: + - chart-example.local + ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## Or for k8s > 1.19 + # - path: /* + # pathType: Prefix + # backend: + # service: + # name: ssl-redirect + # port: + # name: use-annotation + + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Topology Spread Constraints +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## +topologySpreadConstraints: [] + +## Additional init containers (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +## +extraInitContainers: [] + +## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod +extraContainers: "" +# extraContainers: | +# - name: proxy +# image: quay.io/gambol99/keycloak-proxy:latest +# args: +# - -provider=github +# - -client-id= +# - -client-secret= +# - -github-org= +# - -email-domain=* +# - -cookie-secret= +# - -http-address=http://0.0.0.0:4181 +# - -upstream-url=http://127.0.0.1:3000 +# ports: +# - name: proxy-web +# containerPort: 4181 + +## Volumes that can be used in init containers that will not be mounted to deployment pods +extraContainerVolumes: [] +# - name: volume-from-secret +# secret: +# secretName: secret-to-mount +# - name: empty-dir-volume +# emptyDir: {} + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + type: pvc + enabled: false + # storageClassName: default + accessModes: + - ReadWriteOnce + size: 10Gi + # annotations: {} + finalizers: + - kubernetes.io/pvc-protection + # selectorLabels: {} + ## Sub-directory of the PV to mount. Can be templated. + # subPath: "" + ## Name of an existing PVC. Can be templated. + # existingClaim: + ## Extra labels to apply to a PVC. + extraPvcLabels: {} + + ## If persistence is not enabled, this allows to mount the + ## local storage in-memory to improve performance + ## + inMemory: + enabled: false + ## The maximum usage on memory medium EmptyDir would be + ## the minimum value between the SizeLimit specified + ## here and the sum of memory limits of all containers in a pod + ## + # sizeLimit: 300Mi + +initChownData: + ## If false, data ownership will not be reset at startup + ## This allows the grafana-server to be run with an arbitrary user + ## + enabled: true + + ## initChownData container image + ## + image: + repository: rancher/mirrored-library-busybox + tag: "1.31.1" + sha: "" + pullPolicy: IfNotPresent + + ## initChownData resource requests and limits + ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + securityContext: + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + capabilities: + add: + - CHOWN + +# Administrator credentials when not using an existing secret (see below) +adminUser: admin +# adminPassword: strongpassword + +# Use an existing secret for the admin user. +admin: + ## Name of the secret. Can be templated. + existingSecret: "" + userKey: admin-user + passwordKey: admin-password + +## Define command to be executed at startup by grafana container +## Needed if using `vault-env` to manage secrets (ref: https://banzaicloud.com/blog/inject-secrets-into-pods-vault/) +## Default is "run.sh" as defined in grafana's Dockerfile +# command: +# - "sh" +# - "/run.sh" + +## Optionally define args if command is used +## Needed if using `hashicorp/envconsul` to manage secrets +## By default no arguments are set +# args: +# - "-secret" +# - "secret/grafana" +# - "./grafana" + +## Extra environment variables that will be pass onto deployment pods +## +## to provide grafana with access to CloudWatch on AWS EKS: +## 1. create an iam role of type "Web identity" with provider oidc.eks.* (note the provider for later) +## 2. edit the "Trust relationships" of the role, add a line inside the StringEquals clause using the +## same oidc eks provider as noted before (same as the existing line) +## also, replace NAMESPACE and prometheus-operator-grafana with the service account namespace and name +## +## "oidc.eks.us-east-1.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:sub": "system:serviceaccount:NAMESPACE:prometheus-operator-grafana", +## +## 3. attach a policy to the role, you can use a built in policy called CloudWatchReadOnlyAccess +## 4. use the following env: (replace 123456789000 and iam-role-name-here with your aws account number and role name) +## +## env: +## AWS_ROLE_ARN: arn:aws:iam::123456789000:role/iam-role-name-here +## AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +## AWS_REGION: us-east-1 +## +## 5. uncomment the EKS section in extraSecretMounts: below +## 6. uncomment the annotation section in the serviceAccount: above +## make sure to replace arn:aws:iam::123456789000:role/iam-role-name-here with your role arn + +env: {} + +## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core +## Renders in container spec as: +## env: +## ... +## - name: +## valueFrom: +## +envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## The name of a secret in the same kubernetes namespace which contain values to be added to the environment +## This can be useful for auth tokens, etc. Value is templated. +envFromSecret: "" + +## Sensible environment variables that will be rendered as new secret object +## This can be useful for auth tokens, etc. +## If the secret values contains "{{", they'll need to be properly escaped so that they are not interpreted by Helm +## ref: https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function +envRenderSecret: {} + +## The names of secrets in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the secret must be defined with an optional key. +## Name is templated. +envFromSecrets: [] +## - name: secret-name +## prefix: prefix +## optional: true + +## The names of conifgmaps in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the configmap must be defined with an optional key. +## Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#configmapenvsource-v1-core +envFromConfigMaps: [] +## - name: configmap-name +## prefix: prefix +## optional: true + +# Inject Kubernetes services as environment variables. +# See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables +enableServiceLinks: true + +## Additional grafana server secret mounts +# Defines additional mounts with secrets. Secrets must be manually created in the namespace. +extraSecretMounts: [] + # - name: secret-files + # mountPath: /etc/secrets + # secretName: grafana-secret-files + # readOnly: true + # subPath: "" + # + # for AWS EKS (cloudwatch) use the following (see also instruction in env: above) + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # projected: + # defaultMode: 420 + # sources: + # - serviceAccountToken: + # audience: sts.amazonaws.com + # expirationSeconds: 86400 + # path: token + # + # for CSI e.g. Azure Key Vault use the following + # - name: secrets-store-inline + # mountPath: /run/secrets + # readOnly: true + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "akv-grafana-spc" + # nodePublishSecretRef: # Only required when using service principal mode + # name: grafana-akv-creds # Only required when using service principal mode + +## Additional grafana server volume mounts +# Defines additional volume mounts. +extraVolumeMounts: [] + # - name: extra-volume-0 + # mountPath: /mnt/volume0 + # readOnly: true + # - name: extra-volume-1 + # mountPath: /mnt/volume1 + # readOnly: true + # - name: grafana-secrets + # mountPath: /mnt/volume2 + +## Additional Grafana server volumes +extraVolumes: [] + # - name: extra-volume-0 + # existingClaim: volume-claim + # - name: extra-volume-1 + # hostPath: + # path: /usr/shared/ + # type: "" + # - name: grafana-secrets + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "grafana-env-spc" + +## Container Lifecycle Hooks. Execute a specific bash command or make an HTTP request +lifecycleHooks: {} + # postStart: + # exec: + # command: [] + +## Pass the plugins you want installed as a list. +## +plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + ## You can also use other plugin download URL, as long as they are valid zip files, + ## and specify the name of the plugin after the semicolon. Like this: + # - https://grafana.com/api/plugins/marcusolsson-json-datasource/versions/1.3.2/download;marcusolsson-json-datasource + +## Configure grafana datasources +## ref: http://docs.grafana.org/administration/provisioning/#datasources +## +datasources: {} +# datasources.yaml: +# apiVersion: 1 +# datasources: +# - name: Prometheus +# type: prometheus +# url: http://prometheus-prometheus-server +# access: proxy +# isDefault: true +# - name: CloudWatch +# type: cloudwatch +# access: proxy +# uid: cloudwatch +# editable: false +# jsonData: +# authType: default +# defaultRegion: us-east-1 +# deleteDatasources: [] +# - name: Prometheus + +## Configure grafana alerting (can be templated) +## ref: http://docs.grafana.org/administration/provisioning/#alerting +## +alerting: {} + # rules.yaml: + # apiVersion: 1 + # groups: + # - orgId: 1 + # name: '{{ .Chart.Name }}_my_rule_group' + # folder: my_first_folder + # interval: 60s + # rules: + # - uid: my_id_1 + # title: my_first_rule + # condition: A + # data: + # - refId: A + # datasourceUid: '-100' + # model: + # conditions: + # - evaluator: + # params: + # - 3 + # type: gt + # operator: + # type: and + # query: + # params: + # - A + # reducer: + # type: last + # type: query + # datasource: + # type: __expr__ + # uid: '-100' + # expression: 1==0 + # intervalMs: 1000 + # maxDataPoints: 43200 + # refId: A + # type: math + # dashboardUid: my_dashboard + # panelId: 123 + # noDataState: Alerting + # for: 60s + # annotations: + # some_key: some_value + # labels: + # team: sre_team_1 + # contactpoints.yaml: + # secret: + # apiVersion: 1 + # contactPoints: + # - orgId: 1 + # name: cp_1 + # receivers: + # - uid: first_uid + # type: pagerduty + # settings: + # integrationKey: XXX + # severity: critical + # class: ping failure + # component: Grafana + # group: app-stack + # summary: | + # {{ `{{ include "default.message" . }}` }} + +## Configure notifiers +## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels +## +notifiers: {} +# notifiers.yaml: +# notifiers: +# - name: email-notifier +# type: email +# uid: email1 +# # either: +# org_id: 1 +# # or +# org_name: Main Org. +# is_default: true +# settings: +# addresses: an_email_address@example.com +# delete_notifiers: + +## Configure grafana dashboard providers +## ref: http://docs.grafana.org/administration/provisioning/#dashboards +## +## `path` must be /var/lib/grafana/dashboards/ +## +dashboardProviders: {} +# dashboardproviders.yaml: +# apiVersion: 1 +# providers: +# - name: 'default' +# orgId: 1 +# folder: '' +# type: file +# disableDeletion: false +# editable: true +# options: +# path: /var/lib/grafana/dashboards/default + +## Configure grafana dashboard to import +## NOTE: To use dashboards you must also enable/configure dashboardProviders +## ref: https://grafana.com/dashboards +## +## dashboards per provider, use provider name as key. +## +dashboards: {} + # default: + # some-dashboard: + # json: | + # $RAW_JSON + # custom-dashboard: + # file: dashboards/custom-dashboard.json + # prometheus-stats: + # gnetId: 2 + # revision: 2 + # datasource: Prometheus + # local-dashboard: + # url: https://example.com/repository/test.json + # token: '' + # local-dashboard-base64: + # url: https://example.com/repository/test-b64.json + # token: '' + # b64content: true + # local-dashboard-gitlab: + # url: https://example.com/repository/test-gitlab.json + # gitlabToken: '' + # local-dashboard-bitbucket: + # url: https://example.com/repository/test-bitbucket.json + # bearerToken: '' + # local-dashboard-azure: + # url: https://example.com/repository/test-azure.json + # basic: '' + # acceptHeader: '*/*' + +## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. +## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. +## ConfigMap data example: +## +## data: +## example-dashboard.json: | +## RAW_JSON +## +dashboardsConfigMaps: {} +# default: "" + +## Grafana's primary configuration +## NOTE: values in map will be converted to ini format +## ref: http://docs.grafana.org/installation/configuration/ +## +grafana.ini: + paths: + data: /var/lib/grafana/ + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net + server: + domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ .Values.ingress.hosts | first }}{{ else }}''{{ end }}" +## grafana Authentication can be enabled with the following values on grafana.ini + # server: + # The full public facing url you use in browser, used for redirects and emails + # root_url: + # https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana + # auth.github: + # enabled: false + # allow_sign_up: false + # scopes: user:email,read:org + # auth_url: https://github.com/login/oauth/authorize + # token_url: https://github.com/login/oauth/access_token + # api_url: https://api.github.com/user + # team_ids: + # allowed_organizations: + # client_id: + # client_secret: +## LDAP Authentication can be enabled with the following values on grafana.ini +## NOTE: Grafana will fail to start if the value for ldap.toml is invalid + # auth.ldap: + # enabled: true + # allow_sign_up: true + # config_file: /etc/grafana/ldap.toml + +## Grafana's LDAP configuration +## Templated by the template in _helpers.tpl +## NOTE: To enable the grafana.ini must be configured with auth.ldap.enabled +## ref: http://docs.grafana.org/installation/configuration/#auth-ldap +## ref: http://docs.grafana.org/installation/ldap/#configuration +ldap: + enabled: false + # `existingSecret` is a reference to an existing secret containing the ldap configuration + # for Grafana in a key `ldap-toml`. + existingSecret: "" + # `config` is the content of `ldap.toml` that will be stored in the created secret + config: "" + # config: |- + # verbose_logging = true + + # [[servers]] + # host = "my-ldap-server" + # port = 636 + # use_ssl = true + # start_tls = false + # ssl_skip_verify = false + # bind_dn = "uid=%s,ou=users,dc=myorg,dc=com" + +## Grafana's SMTP configuration +## NOTE: To enable, grafana.ini must be configured with smtp.enabled +## ref: http://docs.grafana.org/installation/configuration/#smtp +smtp: + # `existingSecret` is a reference to an existing secret containing the smtp configuration + # for Grafana. + existingSecret: "" + userKey: "user" + passwordKey: "password" + +## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders +## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards +sidecar: + image: + repository: rancher/mirrored-kiwigrid-k8s-sidecar + tag: 1.26.1 + sha: "" + imagePullPolicy: IfNotPresent + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + # skipTlsVerify Set to true to skip tls verification for kube api calls + # skipTlsVerify: true + enableUniqueFilenames: false + readinessProbe: {} + livenessProbe: {} + # Log level default for all sidecars. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. Defaults to INFO + # logLevel: INFO + alerts: + enabled: false + # Additional environment variables for the alerts sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with alert are marked with + label: grafana_alert + # value of label that the configmaps with alert are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for alert config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/alerting/reload" + # Absolute path to shell script to execute after a alert got reloaded + script: null + skipReload: false + # This is needed if skipReload is true, to load any alerts defined at startup time. + # Deploy the alert sidecar as an initContainer. + initAlerts: false + # Additional alert sidecar volume mounts + extraMounts: [] + # Sets the size limit of the alert sidecar emptyDir volume + sizeLimit: {} + dashboards: + enabled: false + # Additional environment variables for the dashboards sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + SCProvider: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + # value of label that the configmaps with dashboards are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) + folder: /tmp/dashboards + # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead + defaultFolderName: null + # Namespaces list. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces. + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # If specified, the sidecar will look for annotation with this name to create folder and put graph here. + # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. + folderAnnotation: null + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/dashboards/reload" + # Absolute path to shell script to execute after a configmap got reloaded + script: null + skipReload: false + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # provider configuration that lets grafana manage the dashboards + provider: + # name of the provider, should be unique + name: sidecarProvider + # orgid as configured in grafana + orgid: 1 + # folder in which the dashboards should be imported in grafana + folder: '' + # type of the provider + type: file + # disableDelete to activate a import-only behaviour + disableDelete: false + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + # allow Grafana to replicate dashboard structure from filesystem + foldersFromFilesStructure: false + # Additional dashboard sidecar volume mounts + extraMounts: [] + # Sets the size limit of the dashboard sidecar emptyDir volume + sizeLimit: {} + datasources: + enabled: false + # Additional environment variables for the datasourcessidecar + env: {} + envValueFrom: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with datasources are marked with + label: grafana_datasource + # value of label that the configmaps with datasources are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for datasource config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload datasources + reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + # Absolute path to shell script to execute after a datasource got reloaded + script: null + skipReload: true + # This is needed if skipReload is true, to load any datasources defined at startup time. + # Deploy the datasources sidecar as an initContainer. + initDatasources: true + # Sets the size limit of the datasource sidecar emptyDir volume + sizeLimit: {} + plugins: + enabled: false + # Additional environment variables for the plugins sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with plugins are marked with + label: grafana_plugin + # value of label that the configmaps with plugins are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for plugin config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload plugins + reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload" + # Absolute path to shell script to execute after a plugin got reloaded + script: null + skipReload: false + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any plugins defined at startup time. + initPlugins: false + # Sets the size limit of the plugin sidecar emptyDir volume + sizeLimit: {} + notifiers: + enabled: false + # Additional environment variables for the notifierssidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with notifiers are marked with + label: grafana_notifier + # value of label that the configmaps with notifiers are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for notifier config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload notifiers + reloadURL: "http://localhost:3000/api/admin/provisioning/notifications/reload" + # Absolute path to shell script to execute after a notifier got reloaded + script: null + skipReload: false + # Deploy the notifier sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any notifiers defined at startup time. + initNotifiers: false + # Sets the size limit of the notifier sidecar emptyDir volume + sizeLimit: {} + +## Override the deployment namespace +## +namespaceOverride: "" + +## Number of old ReplicaSets to retain +## +revisionHistoryLimit: 10 + +## Add a seperate remote image renderer deployment/service +imageRenderer: + deploymentStrategy: {} + # Enable the image-renderer deployment & service + enabled: false + replicas: 1 + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + image: + # image-renderer Image repository + repository: rancher/mirrored-grafana-grafana-image-renderer + # image-renderer Image tag + tag: 3.10.1 + # image-renderer Image sha (optional) + sha: "" + # image-renderer ImagePullPolicy + pullPolicy: Always + # extra environment variables + env: + HTTP_HOST: "0.0.0.0" + # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758 + # RENDERING_MODE: clustered + # IGNORE_HTTPS_ERRORS: true + + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + + # image-renderer deployment serviceAccount + serviceAccountName: "" + # image-renderer deployment securityContext + securityContext: {} + # image-renderer deployment container securityContext + containerSecurityContext: + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ['ALL'] + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ## image-renderer pod annotation + podAnnotations: {} + # image-renderer deployment Host Aliases + hostAliases: [] + # image-renderer deployment priority class + priorityClassName: '' + service: + # Enable the image-renderer service + enabled: true + # image-renderer service port name + portName: 'http' + # image-renderer service port used by both service and deployment + port: 8081 + targetPort: 8081 + # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 1m + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + # See: https://doc.crds.dev/github.com/prometheus-operator/kube-prometheus/monitoring.coreos.com/ServiceMonitor/v1@v0.11.0#spec-targetLabels + targetLabels: [] + # - targetLabel1 + # - targetLabel2 + # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana + grafanaProtocol: http + # In case a sub_path is used this needs to be added to the image renderer callback + grafanaSubPath: "" + # name of the image-renderer port on the pod + podPortName: http + # number of image-renderer replica sets to keep + revisionHistoryLimit: 10 + networkPolicy: + # Enable a NetworkPolicy to limit inbound traffic to only the created grafana pods + limitIngress: true + # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods + limitEgress: false + # Allow additional services to access image-renderer (eg. Prometheus operator when ServiceMonitor is enabled) + extraIngressSelectors: [] + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment (evaluated as template) + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: "default-scheduler" + +networkPolicy: + ## @param networkPolicy.enabled Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to grafana port defined. + ## When true, grafana will accept connections from any source + ## (with the correct destination port). + ## + ingress: true + ## @param networkPolicy.ingress When true enables the creation + ## an ingress network policy + ## + allowExternal: true + ## @param networkPolicy.explicitNamespacesSelector A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed + ## If explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the grafana. + ## But sometimes, we want the grafana to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + ## + ## + ## + ## + ## + ## + egress: + ## @param networkPolicy.egress.enabled When enabled, an egress network policy will be + ## created allowing grafana to connect to external data sources from kubernetes cluster. + enabled: false + ## + ## @param networkPolicy.egress.blockDNSResolution When enabled, DNS resolution will be blocked + ## for all pods in the grafana namespace. + blockDNSResolution: false + ## + ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress + ports: [] + ## Add ports to the egress by specifying - port: + ## E.X. + ## - port: 80 + ## - port: 443 + ## + ## @param networkPolicy.egress.to Allow egress traffic to specific destinations + to: [] + ## Add destinations to the egress by specifying - ipBlock: + ## E.X. + ## to: + ## - namespaceSelector: + ## matchExpressions: + ## - {key: role, operator: In, values: [grafana]} + ## + ## + ## + ## + ## + +# Enable backward compatibility of kubernetes where version below 1.13 doesn't have the enableServiceLinks option +enableKubeBackwardCompatibility: false +useStatefulSet: false +# Create a dynamic manifests via values: +extraObjects: [] + # - apiVersion: "kubernetes-client.io/v1" + # kind: ExternalSecret + # metadata: + # name: grafana-secrets + # spec: + # backendType: gcpSecretsManager + # data: + # - key: grafana-admin-password + # name: adminPassword + +# assertNoLeakedSecrets is a helper function defined in _helpers.tpl that checks if secret +# values are not exposed in the rendered grafana.ini configmap. It is enabled by default. +# +# To pass values into grafana.ini without exposing them in a configmap, use variable expansion: +# https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion +# +# Alternatively, if you wish to allow secret values to be exposed in the rendered grafana.ini configmap, +# you can disable this check by setting assertNoLeakedSecrets to false. +assertNoLeakedSecrets: true diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/Chart.yaml new file mode 100644 index 0000000000..acd648a5b4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedKubelet +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/Chart.yaml new file mode 100644 index 0000000000..068932bacb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedNodeExporter +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/Chart.yaml new file mode 100644 index 0000000000..275e02e5dc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: k3sServer +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/Chart.yaml new file mode 100644 index 0000000000..002a6a180d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/Chart.yaml @@ -0,0 +1,32 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-kube-state-metrics +apiVersion: v2 +appVersion: 2.10.1 +description: Install kube-state-metrics to generate and expose cluster-level metrics +home: https://github.com/kubernetes/kube-state-metrics/ +keywords: +- metric +- monitoring +- prometheus +- kubernetes +maintainers: +- email: tariq.ibrahim@mulesoft.com + name: tariq1890 +- email: manuel@rueg.eu + name: mrueg +- email: david@0xdc.me + name: dotdc +name: kube-state-metrics +sources: +- https://github.com/kubernetes/kube-state-metrics/ +type: application +version: 5.16.4 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/README.md new file mode 100644 index 0000000000..843be89e69 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/README.md @@ -0,0 +1,85 @@ +# kube-state-metrics Helm Chart + +Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics). + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Migrating from stable/kube-state-metrics and kubernetes/kube-state-metrics + +You can upgrade in-place: + +1. [get repository info](#get-repository-info) +1. [upgrade](#upgrading-chart) your existing release name using the new chart repository + +## Upgrading to v3.0.0 + +v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side. + +The upgraded chart now the following changes: + +* Dropped support for helm v2 (helm v3 or later is required) +* collectors key was renamed to resources +* namespace key was renamed to namespaces + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-state-metrics +``` + +### kube-rbac-proxy + +You can enable `kube-state-metrics` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy one RBAC proxy container per endpoint (metrics & telemetry). +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics-read +rules: + - apiGroups: [ "" ] + resources: ["services/kube-state-metrics"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt new file mode 100644 index 0000000000..3589c24ec3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt @@ -0,0 +1,23 @@ +kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. +The exposed metrics can be found here: +https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics + +The metrics are exported on the HTTP endpoint /metrics on the listening port. +In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-state-metrics.namespace" . }}.svc.cluster.local:{{ .Values.service.port }}/metrics + +They are served either as plaintext or protobuf depending on the Accept header. +They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint. + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints are now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "kube-state-metrics.fullname" . }}"] + verbs: + - get +``` +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl new file mode 100644 index 0000000000..ed277fbb53 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl @@ -0,0 +1,196 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "kube-state-metrics.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kube-state-metrics.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kube-state-metrics.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "kube-state-metrics.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-state-metrics.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kube-state-metrics.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "kube-state-metrics.labels" }} +helm.sh/chart: {{ template "kube-state-metrics.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" . }} +{{- include "kube-state-metrics.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kube-state-metrics.selectorLabels" }} +{{- if .Values.selectorOverride }} +{{ toYaml .Values.selectorOverride }} +{{- else }} +app.kubernetes.io/name: {{ include "kube-state-metrics.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "kube-state-metrics.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +The image to use for kube-state-metrics +*/}} +{{- define "kube-state-metrics.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +The image to use for kubeRBACProxy +*/}} +{{- define "kubeRBACProxy.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.kubeRBACProxy.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..025cd47a88 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + endpointSelector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + egress: + {{- if and .Values.networkPolicy.cilium .Values.networkPolicy.cilium.kubeApiServerSelector }} + {{ toYaml .Values.networkPolicy.cilium.kubeApiServerSelector | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + - port: {{ .Values.service.port | quote }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + - port: {{ .Values.selfMonitor.telemetryPort | default 8081 | quote }} + protocol: TCP + {{ end }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..cf9f628d04 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rbac.useClusterRole -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml new file mode 100644 index 0000000000..d38a75a51d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.customResourceState.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config.yaml: | + {{- toYaml .Values.customResourceState.config | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml new file mode 100644 index 0000000000..03158eb948 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml @@ -0,0 +1,314 @@ +apiVersion: apps/v1 +{{- if .Values.autosharding.enabled }} +kind: StatefulSet +{{- else }} +kind: Deployment +{{- end }} +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: +{{ toYaml .Values.annotations | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + replicas: {{ .Values.replicas }} + {{- if not .Values.autosharding.enabled }} + strategy: + type: {{ .Values.updateStrategy | default "RollingUpdate" }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.autosharding.enabled }} + serviceName: {{ template "kube-state-metrics.fullname" . }} + volumeClaimTemplates: [] + {{- end }} + template: + metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 8 }} + {{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + spec: + hostNetwork: {{ .Values.hostNetwork }} + serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 6 }} + {{- end }} + containers: + {{- $servicePort := ternary 9090 (.Values.service.port | default 8080) .Values.kubeRBACProxy.enabled}} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - name: {{ template "kube-state-metrics.name" . }} + {{- if .Values.autosharding.enabled }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + args: + {{- if .Values.extraArgs }} + {{- .Values.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --port={{ $servicePort }} + {{- if .Values.collectors }} + - --resources={{ .Values.collectors | join "," }} + {{- end }} + {{- if .Values.metricLabelsAllowlist }} + - --metric-labels-allowlist={{ .Values.metricLabelsAllowlist | join "," }} + {{- end }} + {{- if .Values.metricAnnotationsAllowList }} + - --metric-annotations-allowlist={{ .Values.metricAnnotationsAllowList | join "," }} + {{- end }} + {{- if .Values.metricAllowlist }} + - --metric-allowlist={{ .Values.metricAllowlist | join "," }} + {{- end }} + {{- if .Values.metricDenylist }} + - --metric-denylist={{ .Values.metricDenylist | join "," }} + {{- end }} + {{- $namespaces := list }} + {{- if .Values.namespaces }} + {{- range $ns := join "," .Values.namespaces | split "," }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + {{- if .Values.releaseNamespace }} + {{- $namespaces = append $namespaces ( include "kube-state-metrics.namespace" . ) }} + {{- end }} + {{- if $namespaces }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + {{- if .Values.namespacesDenylist }} + - --namespaces-denylist={{ tpl (.Values.namespacesDenylist | join ",") $ }} + {{- end }} + {{- if .Values.autosharding.enabled }} + - --pod=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + {{- end }} + {{- if .Values.kubeconfig.enabled }} + - --kubeconfig=/opt/k8s/.kube/config + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - --telemetry-host=127.0.0.1 + - --telemetry-port={{ $telemetryPort }} + {{- else }} + {{- if .Values.selfMonitor.telemetryHost }} + - --telemetry-host={{ .Values.selfMonitor.telemetryHost }} + {{- end }} + {{- if .Values.selfMonitor.telemetryPort }} + - --telemetry-port={{ $telemetryPort }} + {{- end }} + {{- end }} + {{- if .Values.customResourceState.enabled }} + - --custom-resource-state-config-file=/etc/customresourcestate/config.yaml + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumeMounts) }} + volumeMounts: + {{- if .Values.kubeconfig.enabled }} + - name: kubeconfig + mountPath: /opt/k8s/.kube/ + readOnly: true + {{- end }} + {{- if .Values.customResourceState.enabled }} + - name: customresourcestate-config + mountPath: /etc/customresourcestate + readOnly: true + {{- end }} + {{- if .Values.volumeMounts }} +{{ toYaml .Values.volumeMounts | indent 8 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + image: {{ include "kube-state-metrics.image" . }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + {{- if .Values.selfMonitor.enabled }} + - containerPort: {{ $telemetryPort }} + name: "metrics" + {{- end }} + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: /healthz + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- if .Values.resources }} + resources: +{{ toYaml .Values.resources | indent 10 }} +{{- end }} +{{- if .Values.containerSecurityContext }} + securityContext: +{{ toYaml .Values.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-http + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port | default 8080}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.selfMonitor.enabled }} + - name: kube-rbac-proxy-telemetry + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.selfMonitor.telemetryPort | default 8081 }} + - --upstream=http://127.0.0.1:{{ $telemetryPort }}/ + - --proxy-endpoints-port=8889 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + name: "metrics" + - containerPort: 8889 + name: "metrics-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8889 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- end }} + {{- end }} + {{- with .Values.containers }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.topologySpreadConstraints | indent 8 }} + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumes) (.Values.kubeRBACProxy.enabled) }} + volumes: + {{- if .Values.kubeconfig.enabled}} + - name: kubeconfig + secret: + secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + {{- end }} + {{- if .Values.kubeRBACProxy.enabled}} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + {{- end }} + {{- if .Values.customResourceState.enabled}} + - name: customresourcestate-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + {{- end }} + {{- if .Values.volumes }} +{{ toYaml .Values.volumes | indent 8 }} + {{- end }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml new file mode 100644 index 0000000000..6af0084502 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.kubeconfig.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +type: Opaque +data: + config: '{{ .Values.kubeconfig.secret }}' +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml new file mode 100644 index 0000000000..309b38ec54 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "kubernetes") }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + {{- if .Values.networkPolicy.egress }} + ## Deny all egress by default + egress: + {{- toYaml .Values.networkPolicy.egress | nindent 4 }} + {{- end }} + ingress: + {{- if .Values.networkPolicy.ingress }} + {{- toYaml .Values.networkPolicy.ingress | nindent 4 }} + {{- else }} + ## Allow ingress on default ports by default + - ports: + - port: {{ .Values.service.port | default 8080 }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - port: {{ $telemetryPort }} + protocol: TCP + {{- end }} + {{- end }} + podSelector: + {{- if .Values.networkPolicy.podSelector }} + {{- toYaml .Values.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + policyTypes: + - Ingress + - Egress +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml new file mode 100644 index 0000000000..3771b511de --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podDisruptionBudget -}} +{{ if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" -}} +apiVersion: policy/v1 +{{- else -}} +apiVersion: policy/v1beta1 +{{- end }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..d9d944d740 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +{{- if .Values.podSecurityPolicy.annotations }} + annotations: +{{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }} +{{- end }} +spec: + privileged: false + volumes: + - 'secret' +{{- if .Values.podSecurityPolicy.additionalVolumes }} +{{ toYaml .Values.podSecurityPolicy.additionalVolumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..c69e01a716 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..df81c49028 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml new file mode 100644 index 0000000000..671dc9d660 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml @@ -0,0 +1,22 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "kube-state-metrics.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "kube-state-metrics.fullname" . }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/role.yaml new file mode 100644 index 0000000000..0170878376 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/role.yaml @@ -0,0 +1,215 @@ +{{- if not (kindIs "slice" .Values.collectors) }} +{{- fail "Collectors need to be a List since kube-state-metrics chart 3.2.2. Please check README for more information."}} +{{- end }} +{{- if and (eq .Values.rbac.create true) (not .Values.rbac.useExistingRole) -}} +{{- range (ternary (join "," .Values.namespaces | split "," ) (list "") (eq $.Values.rbac.useClusterRole false)) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +{{- if eq $.Values.rbac.useClusterRole false }} +kind: Role +{{- else }} +kind: ClusterRole +{{- end }} +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- if eq $.Values.rbac.useClusterRole false }} + namespace: {{ . }} +{{- end }} +rules: +{{ if has "certificatesigningrequests" $.Values.collectors }} +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] +{{ end -}} +{{ if has "configmaps" $.Values.collectors }} +- apiGroups: [""] + resources: + - configmaps + verbs: ["list", "watch"] +{{ end -}} +{{ if has "cronjobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - cronjobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "daemonsets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "deployments" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpoints" $.Values.collectors }} +- apiGroups: [""] + resources: + - endpoints + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpointslices" $.Values.collectors }} +- apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: ["list", "watch"] +{{ end -}} +{{ if has "horizontalpodautoscalers" $.Values.collectors }} +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "ingresses" $.Values.collectors }} +- apiGroups: ["extensions", "networking.k8s.io"] + resources: + - ingresses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "jobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "leases" $.Values.collectors }} +- apiGroups: ["coordination.k8s.io"] + resources: + - leases + verbs: ["list", "watch"] +{{ end -}} +{{ if has "limitranges" $.Values.collectors }} +- apiGroups: [""] + resources: + - limitranges + verbs: ["list", "watch"] +{{ end -}} +{{ if has "mutatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "namespaces" $.Values.collectors }} +- apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch"] +{{ end -}} +{{ if has "networkpolicies" $.Values.collectors }} +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: ["list", "watch"] +{{ end -}} +{{ if has "nodes" $.Values.collectors }} +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumeclaims" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumes" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "poddisruptionbudgets" $.Values.collectors }} +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "pods" $.Values.collectors }} +- apiGroups: [""] + resources: + - pods + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicasets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicationcontrollers" $.Values.collectors }} +- apiGroups: [""] + resources: + - replicationcontrollers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "resourcequotas" $.Values.collectors }} +- apiGroups: [""] + resources: + - resourcequotas + verbs: ["list", "watch"] +{{ end -}} +{{ if has "secrets" $.Values.collectors }} +- apiGroups: [""] + resources: + - secrets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "services" $.Values.collectors }} +- apiGroups: [""] + resources: + - services + verbs: ["list", "watch"] +{{ end -}} +{{ if has "statefulsets" $.Values.collectors }} +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "storageclasses" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "validatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "volumeattachments" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - volumeattachments + verbs: ["list", "watch"] +{{ end -}} +{{- if $.Values.kubeRBACProxy.enabled }} +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] +{{- end }} +{{- if $.Values.customResourceState.enabled }} +- apiGroups: ["apiextensions.k8s.io"] + resources: + - customresourcedefinitions + verbs: ["list", "watch"] +{{- end }} +{{ if $.Values.rbac.extraRules }} +{{ toYaml $.Values.rbac.extraRules }} +{{ end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml new file mode 100644 index 0000000000..330651b73f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.rbac.useClusterRole false) -}} +{{- range (join "," $.Values.namespaces) | split "," }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} + namespace: {{ . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role +{{- if (not $.Values.rbac.useExistingRole) }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- else }} + name: {{ $.Values.rbac.useExistingRole }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" $ }} + namespace: {{ template "kube-state-metrics.namespace" $ }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/service.yaml new file mode 100644 index 0000000000..6c486a662a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/service.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + annotations: + {{- if .Values.prometheusScrape }} + prometheus.io/scrape: '{{ .Values.prometheusScrape }}' + {{- end }} + {{- if .Values.service.annotations }} + {{- toYaml .Values.service.annotations | nindent 4 }} + {{- end }} +spec: + type: "{{ .Values.service.type }}" + ports: + - name: "http" + protocol: TCP + port: {{ .Values.service.port | default 8080}} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port | default 8080}} + {{ if .Values.selfMonitor.enabled }} + - name: "metrics" + protocol: TCP + port: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + targetPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- if .Values.selfMonitor.telemetryNodePort }} + nodePort: {{ .Values.selfMonitor.telemetryNodePort }} + {{- end }} + {{ end }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if .Values.autosharding.enabled }} + clusterIP: None +{{- else if .Values.service.clusterIP }} + clusterIP: "{{ .Values.service.clusterIP }}" +{{- end }} + selector: + {{- include "kube-state-metrics.selectorLabels" . | indent 4 }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml new file mode 100644 index 0000000000..38a93b31d1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- if .Values.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml new file mode 100644 index 0000000000..01ec44e067 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -0,0 +1,126 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + {{- with .Values.prometheus.monitor.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- with .Values.prometheus.monitor.targetLabels }} + targetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | indent 2 }} + {{- if .Values.prometheus.monitor.namespaceSelector }} + namespaceSelector: + matchNames: + {{- with .Values.prometheus.monitor.namespaceSelector }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + endpoints: + - port: http + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + metricRelabelings: + {{- if .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml .Values.prometheus.monitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.selfMonitor.enabled }} + - port: metrics + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml new file mode 100644 index 0000000000..489de147c1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} + resources: + - statefulsets + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml new file mode 100644 index 0000000000..73b37a4f64 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..f46305b517 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml @@ -0,0 +1,44 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-state-metrics.name" . }} + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{ toYaml .Values.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{ toYaml .Values.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + {{- if .Values.autosharding.enabled }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ template "kube-state-metrics.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/values.yaml new file mode 100644 index 0000000000..bc8ee28fda --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/values.yaml @@ -0,0 +1,491 @@ +# Default values for kube-state-metrics. +prometheusScrape: true +image: + registry: docker.io + repository: rancher/mirrored-kube-state-metrics-kube-state-metrics + tag: v2.10.1 + sha: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: "image-pull-secret" + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "" + +# If set to true, this will deploy kube-state-metrics as a StatefulSet and the data +# will be automatically sharded across <.Values.replicas> pods using the built-in +# autodiscovery feature: https://github.com/kubernetes/kube-state-metrics#automated-sharding +# This is an experimental feature and there are no stability guarantees. +autosharding: + enabled: false + +replicas: 1 + +# Change the deployment strategy when autosharding is disabled. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +# The default is "RollingUpdate" as per Kubernetes defaults. +# During a release, 'RollingUpdate' can lead to two running instances for a short period of time while 'Recreate' can create a small gap in data. +# updateStrategy: Recreate + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +# List of additional cli arguments to configure kube-state-metrics +# for example: --enable-gzip-encoding, --log-file, etc. +# all the possible args can be found here: https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md +extraArgs: [] + +service: + port: 8080 + # Default to clusterIP for backward compatibility + type: ClusterIP + nodePort: 0 + loadBalancerIP: "" + # Only allow access to the loadBalancerIP from these IPs + loadBalancerSourceRanges: [] + clusterIP: "" + annotations: {} + +## Additional labels to add to all resources +customLabels: {} + # app: kube-state-metrics + +## Override selector labels +selectorOverride: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +hostNetwork: false + +rbac: + # If true, create & use RBAC resources + create: true + + # Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to it, rolename set here. + # useExistingRole: your-existing-role + + # If set to false - Run without Cluteradmin privs needed - ONLY works if namespace is also set (if useExistingRole is set this name is used as ClusterRole or Role to bind to) + useClusterRole: true + + # Add permissions for CustomResources' apiGroups in Role/ClusterRole. Should be used in conjunction with Custom Resource State Metrics configuration + # Example: + # - apiGroups: ["monitoring.coreos.com"] + # resources: ["prometheuses"] + # verbs: ["list", "watch"] + extraRules: [] + +# Configure kube-rbac-proxy. When enabled, creates one kube-rbac-proxy container per exposed HTTP endpoint (metrics and telemetry if enabled). +# The requests are served through the same service but requests are then HTTPS. +kubeRBACProxy: + enabled: false + image: + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.14.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-prxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + + ## volumeMounts enables mounting custom volumes in rbac-proxy containers + ## Useful for TLS certificates and keys + volumeMounts: [] + # - mountPath: /etc/tls + # name: kube-rbac-proxy-tls + # readOnly: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created, require rbac true + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # Reference to one or more secrets to be used when pulling images + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + imagePullSecrets: [] + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +prometheus: + monitor: + enabled: false + annotations: {} + additionalLabels: {} + namespace: "" + namespaceSelector: [] + jobLabel: "" + targetLabels: [] + podTargetLabels: [] + interval: "" + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + scrapeTimeout: "" + proxyUrl: "" + ## Whether to enable HTTP2 for servicemonitor + # enableHttp2: false + selectorOverride: {} + honorLabels: false + metricRelabelings: [] + relabelings: [] + scheme: "" + ## File to read bearer token for scraping targets + bearerTokenFile: "" + ## Secret to mount to read bearer token for scraping targets. The secret needs + ## to be in the same namespace as the service monitor and accessible by the + ## Prometheus Operator + bearerTokenSecret: {} + # name: secret-name + # key: key-name + tlsConfig: {} + +## Specify if a Pod Security Policy for kube-state-metrics must be created +## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +podSecurityPolicy: + annotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + additionalVolumes: [] + +## Configure network policy for kube-state-metrics +networkPolicy: + enabled: false + # networkPolicy.flavor -- Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + ## Configure the cilium network policy kube-apiserver selector + # cilium: + # kubeApiServerSelector: + # - toEntities: + # - kube-apiserver + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app.kubernetes.io/name: kube-state-metrics + +securityContext: + enabled: true + runAsGroup: 65534 + runAsUser: 65534 + fsGroup: 65534 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +## Specify security settings for a Container +## Allows overrides and additional options compared to (Pod) securityContext +## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +## Affinity settings for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ +affinity: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +## Topology spread constraints for pod assignment +## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +topologySpreadConstraints: [] + +# Annotations to be added to the deployment/statefulset +annotations: {} + +# Annotations to be added to the pod +podAnnotations: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +# Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} + +# Comma-separated list of metrics to be exposed. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricAllowlist: [] + +# Comma-separated list of metrics not to be enabled. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricDenylist: [] + +# Comma-separated list of additional Kubernetes label keys that will be used in the resource's +# labels metric. By default the metric contains only name and namespace labels. +# To include additional labels, provide a list of resource names in their plural form and Kubernetes +# label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. +# A single '*' can be provided per resource instead to allow any labels, but that has +# severe performance implications (Example: '=pods=[*]'). +metricLabelsAllowlist: [] + # - namespaces=[k8s-label-1,k8s-label-n] + +# Comma-separated list of Kubernetes annotations keys that will be used in the resource' +# labels metric. By default the metric contains only name and namespace labels. +# To include additional annotations provide a list of resource names in their plural form and Kubernetes +# annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. +# A single '*' can be provided per resource instead to allow any annotations, but that has +# severe performance implications (Example: '=pods=[*]'). +metricAnnotationsAllowList: [] + # - pods=[k8s-annotation-1,k8s-annotation-n] + +# Available collectors for kube-state-metrics. +# By default, all available resources are enabled, comment out to disable. +collectors: + - certificatesigningrequests + - configmaps + - cronjobs + - daemonsets + - deployments + - endpoints + - horizontalpodautoscalers + - ingresses + - jobs + - leases + - limitranges + - mutatingwebhookconfigurations + - namespaces + - networkpolicies + - nodes + - persistentvolumeclaims + - persistentvolumes + - poddisruptionbudgets + - pods + - replicasets + - replicationcontrollers + - resourcequotas + - secrets + - services + - statefulsets + - storageclasses + - validatingwebhookconfigurations + - volumeattachments + +# Enabling kubeconfig will pass the --kubeconfig argument to the container +kubeconfig: + enabled: false + # base64 encoded kube-config file + secret: + +# Enabling support for customResourceState, will create a configMap including your config that will be read from kube-state-metrics +customResourceState: + enabled: false + # Add (Cluster)Role permissions to list/watch the customResources defined in the config to rbac.extraRules + config: {} + +# Enable only the release namespace for collecting resources. By default all namespaces are collected. +# If releaseNamespace and namespaces are both set a merged list will be collected. +releaseNamespace: false + +# Comma-separated list(string) or yaml list of namespaces to be enabled for collecting resources. By default all namespaces are collected. +namespaces: "" + +# Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, +# only namespaces that are excluded in namespaces-denylist will be used. +namespacesDenylist: "" + +## Override the deployment namespace +## +namespaceOverride: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +## Provide a k8s version to define apiGroups for podSecurityPolicy Cluster Role. +## For example: kubeTargetVersionOverride: 1.14.9 +## +kubeTargetVersionOverride: "" + +# Enable self metrics configuration for service and Service Monitor +# Default values for telemetry configuration can be overridden +# If you set telemetryNodePort, you must also set service.type to NodePort +selfMonitor: + enabled: false + # telemetryHost: 0.0.0.0 + # telemetryPort: 8081 + # telemetryNodePort: 0 + +# Enable vertical pod autoscaler support for kube-state-metrics +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: [] + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# volumeMounts are used to add custom volume mounts to deployment. +# See example below +volumeMounts: [] +# - mountPath: /etc/config +# name: config-volume + +# volumes are used to add custom volumes to deployment +# See example below +volumes: [] +# - configMap: +# name: cm-for-volume +# name: config-volume + +# Extra manifests to deploy as an array +extraManifests: [] + # - apiVersion: v1 + # kind: ConfigMap + # metadata: + # labels: + # name: prometheus-extra + # data: + # extra-data: "value" + +## Containers allows injecting additional containers. +containers: [] + # - name: crd-init + # image: kiwigrid/k8s-sidecar:latest + +## InitContainers allows injecting additional initContainers. +initContainers: [] + # - name: crd-sidecar + # image: kiwigrid/k8s-sidecar:latest + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml new file mode 100644 index 0000000000..ad9fba247a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/Chart.yaml new file mode 100644 index 0000000000..d144d3ee2a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmEtcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/Chart.yaml new file mode 100644 index 0000000000..a1222c4feb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmProxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/Chart.yaml new file mode 100644 index 0000000000..78a44159a9 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmScheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/Chart.yaml new file mode 100644 index 0000000000..d067725a17 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/Chart.yaml @@ -0,0 +1,28 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-prometheus-adapter +apiVersion: v1 +appVersion: v0.10.0 +description: A Helm chart for k8s prometheus adapter +home: https://github.com/kubernetes-sigs/prometheus-adapter +keywords: +- hpa +- metrics +- prometheus +- adapter +kubeVersion: '>=1.26.0-0' +maintainers: +- email: mattias.gees@jetstack.io + name: mattiasgees +- name: steven-sheehy +- email: hfernandez@mesosphere.com + name: hectorj2f +name: prometheus-adapter +sources: +- https://github.com/kubernetes/charts +- https://github.com/kubernetes-sigs/prometheus-adapter +version: 4.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/README.md new file mode 100644 index 0000000000..d77bb0c920 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/README.md @@ -0,0 +1,160 @@ +# Prometheus Adapter + +Installs the [Prometheus Adapter](https://github.com/kubernetes-sigs/prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics. + +## Prerequisites + +Kubernetes 1.14+ + +## Get Helm Repositories Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-adapter +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Helm Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### To 4.2.0 + +Readiness and liveness probes are now fully configurable through values `readinessProbe` and `livenessProbe`. The previous values have been kept as defaults. + +### To 4.0.0 + +Previously, security context of the container was set directly in the deployment template. This release makes it configurable through the new configuration variable `securityContext` whilst keeping the previously set values as defaults. Furthermore, previous variable `runAsUser` is now set in `securityContext` and is not used any longer. Please, use `securityContext.runAsUser` instead. In the same security context, `seccompProfile` has been enabled and set to type `RuntimeDefault`. + +### To 3.0.0 + +Due to a change in deployment labels, the upgrade requires `helm upgrade --force` in order to re-create the deployment. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-adapter +``` + +### Prometheus Service Endpoint + +To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`. + +### Adapter Rules + +Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/kubernetes-sigs/prometheus-adapter/blob/master/docs/config.md) for the proper format). + +### Horizontal Pod Autoscaler Metrics + +Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics). + +The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis): + +### Custom Metrics + +Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`: + +```yaml +rules: + custom: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### External Metrics + +Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`: + +```yaml +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_external_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### Resource Metrics + +Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`: + +```yaml +rules: + resource: + cpu: + containerQuery: | + sum by (<<.GroupBy>>) ( + rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + - + avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m +``` + +**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt new file mode 100644 index 0000000000..b7b9b99322 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt @@ -0,0 +1,9 @@ +{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed. +In a few minutes you should be able to list metrics using the following command(s): +{{ if .Values.rules.resource }} + kubectl get --raw /apis/metrics.k8s.io/v1beta1 +{{- end }} + kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 +{{ if .Values.rules.external }} + kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl new file mode 100644 index 0000000000..edbb829b2b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl @@ -0,0 +1,113 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-prometheus-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-prometheus-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "k8s-prometheus-adapter.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-prometheus-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "k8s-prometheus-adapter.labels" }} +helm.sh/chart: {{ include "k8s-prometheus-adapter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "k8s-prometheus-adapter.name" . }} +{{- include "k8s-prometheus-adapter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "k8s-prometheus-adapter.selectorLabels" }} +app.kubernetes.io/name: {{ include "k8s-prometheus-adapter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8s-prometheus-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "k8s-prometheus-adapter.pdb.apiVersion" -}} +{{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} + {{- print "policy/v1" -}} +{{- else -}} + {{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml new file mode 100644 index 0000000000..4e32c964c6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml @@ -0,0 +1,76 @@ +{{- if .Values.certManager.enabled -}} +--- +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + duration: {{ .Values.certManager.caCertDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.prometheus-adapter" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + ca: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert +--- +# Finally, generate a serving certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + duration: {{ .Values.certManager.certDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + dnsNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }}.svc +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml new file mode 100644 index 0000000000..6701e6ba08 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-system-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml new file mode 100644 index 0000000000..67efd2aa2f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml new file mode 100644 index 0000000000..2c690a03cc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml new file mode 100644 index 0000000000..17f415d970 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml @@ -0,0 +1,97 @@ +{{- if not .Values.rules.existing -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +data: + config.yaml: | +{{- if or .Values.rules.default .Values.rules.custom }} + rules: +{{- if .Values.rules.default }} + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{- end -}} +{{- if .Values.rules.custom }} +{{ toYaml .Values.rules.custom | indent 4 }} +{{- end -}} +{{- end -}} +{{- if .Values.rules.external }} + externalRules: +{{ toYaml .Values.rules.external | indent 4 }} +{{- end -}} +{{- if .Values.rules.resource }} + resourceRules: +{{ toYaml .Values.rules.resource | indent 6 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml new file mode 100644 index 0000000000..8b7b4e555e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if or .Values.rules.default .Values.rules.custom }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: custom.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..0cc6920836 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,24 @@ +{{- /* +This if must be aligned with custom-metrics-cluster-role.yaml +as otherwise this binding will point to not existing role. +*/ -}} +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml new file mode 100644 index 0000000000..4aa15ffe99 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: ["*"] + verbs: ["*"] +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml new file mode 100644 index 0000000000..a7ea3310a0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml @@ -0,0 +1,143 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + {{- if or .Values.customAnnotations .Values.deploymentAnnotations }} + annotations: + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.deploymentAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + replicas: {{ .Values.replicas }} + strategy: {{ toYaml .Values.strategy | nindent 4 }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} + template: + metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + name: {{ template "k8s-prometheus-adapter.name" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy }} + {{- end}} + containers: + - name: {{ .Chart.Name }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 8 }} + {{- end }} + args: + - /adapter + - --secure-port={{ .Values.listenPort }} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - --tls-cert-file=/var/run/serving-cert/tls.crt + - --tls-private-key-file=/var/run/serving-cert/tls.key + {{- end }} + - --cert-dir=/tmp/cert + - --prometheus-url={{ tpl .Values.prometheus.url . }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}}{{ .Values.prometheus.path }} + - --metrics-relist-interval={{ .Values.metricsRelistInterval }} + - --v={{ .Values.logLevel }} + - --config=/etc/adapter/config.yaml + {{- if .Values.extraArguments }} + {{- toYaml .Values.extraArguments | trim | nindent 8 }} + {{- end }} + ports: + - containerPort: {{ .Values.listenPort }} + name: https + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }} + {{ end }} + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} + priorityClassName: {{ .Values.priorityClassName }} + {{- if .Values.podSecurityContext }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | trim | nindent 6 }} + {{ end }} + - name: config + configMap: + name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }} + - name: tmp + emptyDir: {} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - name: volume-serving-cert + secret: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml new file mode 100644 index 0000000000..21339af128 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.external }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: external.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..05547bd323 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml new file mode 100644 index 0000000000..212ea78b25 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml new file mode 100644 index 0000000000..205761a9f1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml @@ -0,0 +1,23 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: {{ include "k8s-prometheus-adapter.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/psp.yaml new file mode 100644 index 0000000000..fded5a7491 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/psp.yaml @@ -0,0 +1,66 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.psp.create) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + hostPorts: + - min: {{ .Values.listenPort }} + max: {{ .Values.listenPort }} + {{- end }} + fsGroup: + rule: RunAsAny + runAsGroup: + rule: RunAsAny + runAsUser: + rule: MustRunAs + ranges: + - min: 1024 + max: 65535 + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - secret + - emptyDir + - configMap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +rules: +- apiGroups: + - 'policy' + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml new file mode 100644 index 0000000000..0cc9fff6a2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.resource}} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml new file mode 100644 index 0000000000..3c247e48d2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml new file mode 100644 index 0000000000..73d8953046 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml new file mode 100644 index 0000000000..d3c77c1c65 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/secret.yaml new file mode 100644 index 0000000000..3e7e8887bd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- if .Values.tls.enable -}} +apiVersion: v1 +kind: Secret +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +type: kubernetes.io/tls +data: + tls.crt: {{ b64enc .Values.tls.certificate }} + tls.key: {{ b64enc .Values.tls.key }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/service.yaml new file mode 100644 index 0000000000..ddac37cfa1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + {{- if or .Values.service.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.service.annotations }} + {{ toYaml .Values.service.annotations | indent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + ports: + - port: {{ .Values.service.port }} + protocol: TCP + targetPort: https + selector: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 4 }} + type: {{ .Values.service.type }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..30a169ae0e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +{{- if or .Values.serviceAccount.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.serviceAccount.annotations }} + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/values.yaml new file mode 100644 index 0000000000..a1445a23f1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/values.yaml @@ -0,0 +1,277 @@ +# Default values for k8s-prometheus-adapter.. +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +affinity: {} + +topologySpreadConstraints: [] + +image: + repository: rancher/mirrored-prometheus-adapter-prometheus-adapter + tag: v0.12.0 + pullPolicy: IfNotPresent + +logLevel: 4 + +metricsRelistInterval: 1m + +listenPort: 6443 + +nodeSelector: {} + +priorityClassName: "" + +## Override the release namespace (for multi-namespace deployments in combined charts) +namespaceOverride: "" + +## Additional annotations to add to all resources +customAnnotations: {} + # role: custom-metrics + +## Additional labels to add to all resources +customLabels: {} + # monitoring: prometheus-adapter + +# Url to access prometheus +prometheus: + # Value is templated + url: http://prometheus.default.svc + port: 9090 + path: "" + +replicas: 1 + +# k8s 1.21 needs fsGroup to be set for non root deployments +# ref: https://github.com/kubernetes/kubernetes/issues/70679 +podSecurityContext: + fsGroup: 10001 + +# SecurityContext of the container +# ref. https://kubernetes.io/docs/tasks/configure-pod-container/security-context +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["all"] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + +rbac: + # Specifies whether RBAC resources should be created + create: true + +psp: + # Specifies whether PSP resources should be created + create: false + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +# Custom DNS configuration to be added to prometheus-adapter pods +dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + +# Configure liveness probe +# https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#Probe +livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +# Configure readiness probe +readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +rules: + default: true + + custom: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_custom_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # Mounts a configMap with pre-generated rules for use. Overrides the + # default, custom, external and resource entries + existing: + + external: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_external_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # resource: + # cpu: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # memory: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + # - + # avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # window: 3m + +service: + annotations: {} + port: 443 + type: ClusterIP + # clusterIP: 1.2.3.4 + +tls: + enable: false + ca: |- + # Public CA file that signed the APIService + key: |- + # Private key of the APIService + certificate: |- + # Public key of the APIService + +# Set environment variables from secrets, configmaps or by setting them as name/value +env: [] + # - name: TMP_DIR + # value: /tmp + # - name: PASSWORD + # valueFrom: + # secretKeyRef: + # name: mysecret + # key: password + # optional: false + +# Any extra arguments +extraArguments: [] + # - --tls-private-key-file=/etc/tls/tls.key + # - --tls-cert-file=/etc/tls/tls.crt + +# Any extra volumes +extraVolumes: [] + # - name: example-name + # hostPath: + # path: /path/on/host + # type: DirectoryOrCreate + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + # type: File + +# Any extra volume mounts +extraVolumeMounts: [] + # - name: example-name + # mountPath: /path/in/container + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +tolerations: [] + +# Labels added to the pod +podLabels: {} + +# Annotations added to the pod +podAnnotations: {} + +# Annotations added to the deployment +deploymentAnnotations: {} + +hostNetwork: + # Specifies if prometheus-adapter should be started in hostNetwork mode. + # + # You would require this enabled if you use alternate overlay networking for pods and + # API server unable to communicate with metrics-server. As an example, this is required + # if you use Weave network on EKS. See also dnsPolicy + enabled: false + +# When hostNetwork is enabled, you probably want to set this to ClusterFirstWithHostNet +# dnsPolicy: ClusterFirstWithHostNet + +# Deployment strategy type +strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% + +podDisruptionBudget: + # Specifies if PodDisruptionBudget should be enabled + # When enabled, minAvailable or maxUnavailable should also be defined. + enabled: false + minAvailable: + maxUnavailable: 1 + +certManager: + enabled: false + caCertDuration: 43800h + certDuration: 8760h diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/Chart.yaml new file mode 100644 index 0000000000..9130cbcc91 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/Chart.yaml @@ -0,0 +1,25 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: |- + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts +apiVersion: v2 +appVersion: 1.7.0 +description: A Helm chart for prometheus node-exporter +home: https://github.com/prometheus/node_exporter/ +keywords: +- node-exporter +- prometheus +- exporter +maintainers: +- email: gianrubio@gmail.com + name: gianrubio +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rootsandtrees@posteo.de + name: zeritti +name: prometheus-node-exporter +sources: +- https://github.com/prometheus/node_exporter/ +type: application +version: 4.30.3 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/README.md new file mode 100644 index 0000000000..149b982267 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/README.md @@ -0,0 +1,97 @@ + +# Prometheus Node Exporter + +Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a Prometheus [Node Exporter](http://github.com/prometheus/node_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-node-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/prometheus-node-exporter --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### 3.x to 4.x + +Starting from version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i prometheus-node-exporter prometheus-community/prometheus-node-exporter +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 2.x to 3.x + +Change the following: + +```yaml +hostRootFsMount: true +``` + +to: + +```yaml +hostRootFsMount: + enabled: true + mountPropagation: HostToContainer +``` + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-node-exporter +``` + +### kube-rbac-proxy + +You can enable `prometheus-node-exporter` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy a RBAC proxy container protecting the node-exporter endpoint. +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-node-exporter-read +rules: + - apiGroups: [ "" ] + resources: ["services/node-exporter-prometheus-node-exporter"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt new file mode 100644 index 0000000000..8c5391f1f7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt @@ -0,0 +1,29 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus-node-exporter.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "prometheus-node-exporter.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "prometheus-node-exporter.namespace" . }} {{ template "prometheus-node-exporter.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "prometheus-node-exporter.namespace" . }} -l "app.kubernetes.io/name={{ template "prometheus-node-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:{{ .Values.service.port }} to use your application" + kubectl port-forward --namespace {{ template "prometheus-node-exporter.namespace" . }} $POD_NAME {{ .Values.service.port }} +{{- end }} + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints is now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "prometheus-node-exporter.fullname" . }}"] + verbs: + - get +``` +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl new file mode 100644 index 0000000000..72a6db45a1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -0,0 +1,236 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus-node-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus-node-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-node-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-node-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-node-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-node-exporter.name" . }} +{{ include "prometheus-node-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-node-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-node-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-node-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-node-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-node-exporter.image" -}} +{{- $temp_registry := (include "system_default_registry" .) }} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if $temp_registry }} +{{- printf "%s%s:%s@%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if $temp_registry }} +{{- printf "%s%s:%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-node-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-node-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-node-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-node-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* Sets sidecar volumeMounts */}} +{{- define "prometheus-node-exporter.sidecarVolumeMounts" -}} +{{- range $_, $mount := $.Values.sidecarVolumeMount }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- end }} +{{- range $_, $mount := $.Values.sidecarHostVolumeMounts }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- if $mount.mountPropagation }} + mountPropagation: {{ $mount.mountPropagation }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml new file mode 100644 index 0000000000..c256dba73d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: + {{- if $.Values.kubeRBACProxy.enabled }} + - apiGroups: [ "authentication.k8s.io" ] + resources: + - tokenreviews + verbs: [ "create" ] + - apiGroups: [ "authorization.k8s.io" ] + resources: + - subjectaccessreviews + verbs: [ "create" ] + {{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..653305ad9e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + name: {{ template "prometheus-node-exporter.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml new file mode 100644 index 0000000000..48d274f1b1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -0,0 +1,309 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ ternary true false (or .Values.serviceAccount.automountServiceAccountToken .Values.kubeRBACProxy.enabled) }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.extraInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-node-exporter.serviceAccountName" . }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + containers: + {{- $servicePort := ternary .Values.kubeRBACProxy.port .Values.service.port .Values.kubeRBACProxy.enabled }} + - name: node-exporter + image: {{ include "prometheus-node-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + {{- if .Values.hostRootFsMount.enabled }} + - --path.rootfs=/host/root + {{- if semverCompare ">=1.4.0-0" (coalesce .Values.version .Values.image.tag .Chart.AppVersion) }} + - --path.udev.data=/host/root/run/udev/data + {{- end }} + {{- end }} + - --web.listen-address=[$(HOST_IP)]:{{ $servicePort }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: HOST_IP + {{- if .Values.kubeRBACProxy.enabled }} + value: 127.0.0.1 + {{- else if .Values.service.listenOnAllInterfaces }} + value: 0.0.0.0 + {{- else }} + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + {{- end }} + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - name: {{ .Values.service.portName }} + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + volumeMounts: + - name: proc + mountPath: /host/proc + {{- with .Values.hostProcFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + - name: sys + mountPath: /host/sys + {{- with .Values.hostSysFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- if .Values.hostRootFsMount.enabled }} + - name: root + mountPath: /host/root + {{- with .Values.hostRootFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- with $mount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- range .Values.sidecars }} + {{- $overwrites := dict "volumeMounts" (concat (include "prometheus-node-exporter.sidecarVolumeMounts" $ | fromYamlArray) (.volumeMounts | default list) | default list) }} + {{- $defaults := dict "image" (include "prometheus-node-exporter.image" $) "securityContext" $.Values.containerSecurityContext "imagePullPolicy" $.Values.image.pullPolicy }} + - {{- toYaml (merge $overwrites . $defaults) | nindent 10 }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 12 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- if .Values.kubeRBACProxy.image.sha }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}@sha256:{{ .Values.kubeRBACProxy.image.sha }}" + {{- else }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}" + {{- end }} + ports: + - containerPort: {{ .Values.service.port}} + name: {{ .Values.kubeRBACProxy.portName }} + {{- if .Values.kubeRBACProxy.enableHostPort }} + hostPort: {{ .Values.service.port }} + {{- end }} + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: + {{- toYaml .Values.kubeRBACProxy.resources | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + {{- with .Values.kubeRBACProxy.env }} + env: + {{- range $key, $value := $.Values.kubeRBACProxy.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: + {{ toYaml .Values.kubeRBACProxy.containerSecurityContext | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.restartPolicy }} + restartPolicy: {{ . }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + {{- if .Values.hostRootFsMount.enabled }} + - name: root + hostPath: + path: / + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- with $mount.type }} + type: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml new file mode 100644 index 0000000000..56b695203a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml @@ -0,0 +1,18 @@ +{{- if .Values.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +subsets: + - addresses: + {{- range .Values.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml new file mode 100644 index 0000000000..2b21b71062 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl . $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml new file mode 100644 index 0000000000..825722729d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml @@ -0,0 +1,23 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingress: + - ports: + - port: {{ .Values.service.port }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml new file mode 100644 index 0000000000..f88da6a34e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.podmonitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-node-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..ee5bbba4a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml @@ -0,0 +1,14 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ include "prometheus-node-exporter.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..160f2bbf7a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ include "prometheus-node-exporter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml new file mode 100644 index 0000000000..f3b52e1120 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml @@ -0,0 +1,49 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.rbac.pspAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + - 'hostPath' + hostNetwork: true + hostIPC: false + hostPID: true + hostPorts: + - min: 0 + max: 65535 + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml new file mode 100644 index 0000000000..814e110337 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + namespace: {{ include "prometheus-node-exporter.namespace" . }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "prometheus-node-exporter.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "prometheus-node-exporter.fullname" . }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml new file mode 100644 index 0000000000..a065e46e39 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml @@ -0,0 +1,29 @@ +{{- if .Values.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.service.ipDualStack.ipFamilyPolicy }} +{{- end }} + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: {{ .Values.service.portName }} + selector: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..5c3348c09b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..6d6e440473 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.monitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + metricRelabelings: + {{- with .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..2c2705f872 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: node-exporter + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ . }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: {{ include "prometheus-node-exporter.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/values.yaml new file mode 100644 index 0000000000..b9f2f7ab87 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/values.yaml @@ -0,0 +1,530 @@ +# Default values for prometheus-node-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-node-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: v1.7.0 + pullPolicy: IfNotPresent + digest: "" + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +global: + cattle: + psp: + enable: true + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "docker.io" + +# Configure kube-rbac-proxy. When enabled, creates a kube-rbac-proxy to protect the node-exporter http endpoint. +# The requests are served through the same service but requests are HTTPS. +kubeRBACProxy: + enabled: false + ## Set environment variables as name/value pairs + env: {} + # VARIABLE: value + image: + registry: docker.io + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.15.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-proxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: {} + + # Specify the port used for the Node exporter container (upstream port) + port: 8100 + # Specify the name of the container port + portName: http + # Configure a hostPort. If true, hostPort will be enabled in the container and set to service.port. + enableHostPort: false + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +service: + enabled: true + type: ClusterIP + port: 9796 + targetPort: 9796 + nodePort: + portName: metrics + listenOnAllInterfaces: true + annotations: + prometheus.io/scrape: "true" + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + +# Set a NetworkPolicy with: +# ingress only on service.port +# no egress permitted +networkPolicy: + enabled: false + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + + jobLabel: "" + + # List of pod labels to add to node exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: [] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Node Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +# Specify the container restart policy passed to the Node Export container +# Possible Values: Always (default)|OnFailure|Never +restartPolicy: null + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + +containerSecurityContext: + readOnlyRootFilesystem: true + # capabilities: + # add: + # - SYS_TIME + +rbac: + ## If true, create & use RBAC resources + ## + create: true + pspAnnotations: {} + +# for deployments that have node_exporter deployed outside of the cluster, list +# their addresses here +endpoints: [] + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +# Mount the node's root file system (/) at /host/root in the container +hostRootFsMount: + enabled: true + # Defines how new mounts in existing mounts on the node or in the container + # are propagated to the container or node, respectively. Possible values are + # None, HostToContainer, and Bidirectional. If this field is omitted, then + # None is used. More information on: + # https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation + mountPropagation: HostToContainer + +# Mount the node's proc file system (/proc) at /host/proc in the container +hostProcFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +# Mount the node's sys file system (/sys) at /host/sys in the container +hostSysFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to node exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to node exporter pods +podLabels: {} + +# Annotations to be added to node exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-node-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: linux + # kubernetes.io/arch: amd64 + +# Specify grace period for graceful termination of pods. Defaults to 30 if null or not specified +terminationGracePeriodSeconds: null + +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + +# Enable or disable container termination message settings +# https://kubernetes.io/docs/tasks/debug/debug-application/determine-reason-pod-failure/ +terminationMessageParams: + enabled: false + # If enabled, specify the path for termination messages + terminationMessagePath: /dev/termination-log + # If enabled, specify the policy for termination messages + terminationMessagePolicy: File + + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.diskstats.ignored-devices=^(ram|loop|fd|(h|s|v)d[a-z]|nvme\\d+n\\d+p)\\d+$ +# - --collector.textfile.directory=/run/prometheus + +## Additional mounts from the host to node-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# https://kubernetes.io/docs/concepts/storage/volumes/#hostpath-volume-types +# type: "" (Default)|DirectoryOrCreate|Directory|FileOrCreate|File|Socket|CharDevice|BlockDevice +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file; fields image,imagePullPolicy,securityContext take default value from main container +## +sidecars: [] +# - name: nvidia-dcgm-exporter +# image: nvidia/dcgm-exporter:1.4.3 +# volumeMounts: +# - name: tmp +# mountPath: /tmp + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +# - name: collector-textfiles +# mountPath: /run/prometheus +# readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +# Enable vertical pod autoscaler support for prometheus-node-exporter +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# Extra manifests to deploy as an array +extraManifests: [] + # - | + # apiVersion: v1 + # kind: ConfigMap + # metadata: + # name: prometheus-extra + # data: + # extra-data: "value" + +# Override version of app, required if image.tag is defined and does not follow semver +version: "" diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/Chart.yaml new file mode 100644 index 0000000000..dbf4d0b815 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2ControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/Chart.yaml new file mode 100644 index 0000000000..87495a6a6b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Etcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/Chart.yaml new file mode 100644 index 0000000000..4bce6d41ea --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2IngressNginx +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/Chart.yaml new file mode 100644 index 0000000000..0a316e05c7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Proxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/Chart.yaml new file mode 100644 index 0000000000..fa6aa5ac66 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Scheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/Chart.yaml new file mode 100644 index 0000000000..df00a46b66 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/Chart.yaml new file mode 100644 index 0000000000..96b33f0bc8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeEtcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/Chart.yaml new file mode 100644 index 0000000000..bc49bcefd2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeIngressNginx +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/Chart.yaml new file mode 100644 index 0000000000..18eac324d4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeProxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/Chart.yaml new file mode 100644 index 0000000000..8c53b63bcc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeScheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/Chart.yaml new file mode 100644 index 0000000000..784bb0ec7e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +appVersion: 0.25.1 +description: A Helm chart for prometheus windows-exporter +home: https://github.com/prometheus-community/windows_exporter/ +keywords: +- windows-exporter +- windows +- prometheus +- exporter +maintainers: +- email: github@jkroepke.de + name: jkroepke +name: windowsExporter +sources: +- https://github.com/prometheus-community/windows_exporter/ +type: application +version: 0.3.1 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/README.md new file mode 100644 index 0000000000..1da1c64e12 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/README.md @@ -0,0 +1,42 @@ +# Prometheus `Windows Exporter` + +Prometheus exporter for hardware and OS metrics exposed by Windows kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a prometheus [`Windows Exporter`](http://github.com/prometheus-community/windows_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-windows-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-windows-exporter +``` diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 new file mode 100644 index 0000000000..9cbed7112d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 @@ -0,0 +1,31 @@ +$ErrorActionPreference = 'Continue' + +function CheckFirewallRuleError { + # We hit an error. This can happen for a number of reasons, including if the rule already exists + if ($error[0]) { + if (($error[0].Exception.NativeErrorCode) -and ($error[0].Exception.NativeErrorCode.ToString() -eq "AlreadyExists")) { + # Previous versions of monitoring may have already created this Firewall Rule + # Because of this, if the rule alreadys exists there is no need to delete and recreate it. + Write-Host "Detected Existing Firewall Rule, Nothing To Do" + } else { + Write-Host "Error Encountered Setting Up Required Firewall Rule" + $error[0].Exception + exit 1 + } + } +} + +Write-Host "Attempting To Configure Firewall Rules For Ports 9796, 10250" + +# This is the exact same firewall rule that has historically been created by rancher-wins +# https://github.com/rancher/wins/blob/91f670c47f19c6d9fe97d8f66a695d3081ad994f/pkg/apis/process_service_mgmt.go#L149 +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-9796 -Name rancher-wins-windows-exporter-TCP-9796 -Action Allow -Protocol TCP -LocalPort 9796 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Node Exporter Firewall Rule Successfully Created" + +# This rule is required in order to have the Rancher UI display node metrics in the 'Nodes' tab of the cluster explorer +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-10250 -Name rancher-wins-windows-exporter-TCP-10250 -Action Allow -Protocol TCP -LocalPort 10250 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Prometheus Metrics Firewall Rule Successfully Created" + +Write-Host "All Firewall Rules Successfully Configured" diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..c9a5d6db8c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/_helpers.tpl @@ -0,0 +1,216 @@ +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "prometheus-windows-exporter.fullname" -}} +{{ printf "%s-windows-exporter" .Release.Name }} +{{- end -}} + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "windowsExporter.renamedMetricsRelabeling" -}} +{{- range $original, $new := (include "windowsExporter.renamedMetrics" . | fromJson) -}} +- sourceLabels: [__name__] + regex: {{ $original }} + replacement: '{{ $new }}' + targetLabel: __name__ +{{ end -}} +{{- end -}} + +{{- define "windowsExporter.labels" -}} +k8s-app: {{ template "prometheus-windows-exporter.fullname" . }} +release: {{ .Release.Name }} +component: "windows-exporter" +provider: kubernetes +{{- end -}} + +{{- define "windowsExporter.renamedMetrics" -}} +{{- $renamed := dict -}} +{{/* v0.15.0 */}} +{{- $_ := set $renamed "windows_mssql_transactions_active_total" "windows_mssql_transactions_active" -}} +{{/* v0.16.0 */}} +{{- $_ := set $renamed "windows_adfs_ad_login_connection_failures" "windows_adfs_ad_login_connection_failures_total" -}} +{{- $_ := set $renamed "windows_adfs_certificate_authentications" "windows_adfs_certificate_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_device_authentications" "windows_adfs_device_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_extranet_account_lockouts" "windows_adfs_extranet_account_lockouts_total" -}} +{{- $_ := set $renamed "windows_adfs_federated_authentications" "windows_adfs_federated_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_passport_authentications" "windows_adfs_passport_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_failed" "windows_adfs_password_change_failed_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_succeeded" "windows_adfs_password_change_succeeded_total" -}} +{{- $_ := set $renamed "windows_adfs_token_requests" "windows_adfs_token_requests_total" -}} +{{- $_ := set $renamed "windows_adfs_windows_integrated_authentications" "windows_adfs_windows_integrated_authentications_total" -}} +{{- $_ := set $renamed "windows_net_packets_outbound_errors" "windows_net_packets_outbound_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_discarded" "windows_net_packets_received_discarded_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_errors" "windows_net_packets_received_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_total" "windows_net_packets_received_total_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_unknown" "windows_net_packets_received_unknown_total" -}} +{{- $_ := set $renamed "windows_dns_memory_used_bytes_total" "windows_dns_memory_used_bytes" -}} +{{- $renamed | toJson -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-windows-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-windows-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-windows-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-windows-exporter.name" . }} +{{ include "prometheus-windows-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-windows-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-windows-exporter.fullname" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-windows-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-windows-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-windows-exporter.image" -}} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-windows-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-windows-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-windows-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-windows-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/config.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/config.yaml new file mode 100644 index 0000000000..25f1fa69c2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + config.yml: | + {{- .Values.config | nindent 4 }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/daemonset.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/daemonset.yaml new file mode 100644 index 0000000000..be7feb3ed1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/daemonset.yaml @@ -0,0 +1,200 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "windowsExporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + initContainers: + - name: configure-firewall + image: {{ include "prometheus-windows-exporter.image" . }} + command: + - C:\WINDOWS\System32\WindowsPowerShell\v1.0\powershell.exe + args: ["-f", "scripts/configure-firewall.ps1"] + volumeMounts: + - mountPath: /scripts + name: exporter-scripts + {{- with .Values.extraInitContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-windows-exporter.fullname" . }} + containers: + - name: windows-exporter + image: {{ include "prometheus-windows-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --config.file=%CONTAINER_SANDBOX_MOUNT_POINT%/config.yml + - --collector.textfile.directories=%CONTAINER_SANDBOX_MOUNT_POINT% + - --web.listen-address=:{{ .Values.service.port }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + hostPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /config.yml + subPath: config.yml + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- if or .Values.sidecarVolumeMount .Values.sidecarHostVolumeMounts }} + volumeMounts: + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: exporter-scripts + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + - name: config + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml new file mode 100644 index 0000000000..bbb6c39340 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.podmonitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-windows-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-windows-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml new file mode 100644 index 0000000000..f514c8161a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{ (.Files.Glob "scripts/*").AsConfig | indent 2 }} + diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/service.yaml new file mode 100644 index 0000000000..267b796f63 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/service.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- if or .Values.prometheus.monitor.enabled .Values.prometheus.podMonitor.enabled }} + {{- with .Values.service.annotations }} + annotations: + {{- unset . "prometheus.io/scrape" | toYaml | nindent 4 }} + {{- end }} + {{- else }} + annotations: + prometheus.io/scrape: "true" + {{- with .Values.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port }} + protocol: TCP + appProtocol: http + name: {{ .Values.service.portName }} + selector: + {{- include "windowsExporter.labels" . | nindent 4 }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..14c1c46807 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-windows-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..2effc07758 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.monitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + metricRelabelings: +{{- include "windowsExporter.renamedMetricsRelabeling" . | nindent 6 -}} + - sourceLabels: [__name__] + regex: 'wmi_(.*)' + replacement: 'windows_$1' + targetLabel: __name__ + - sourceLabels: [volume, nic] + regex: (.*);(.*) + separator: '' + targetLabel: device + action: replace + replacement: $1$2 + - sourceLabels: [__name__] + regex: windows_cs_logical_processors + replacement: 'system' + targetLabel: mode + relabelings: + - separator: ':' + sourceLabels: + - __meta_kubernetes_pod_host_ip + - __meta_kubernetes_pod_container_port_number + targetLabel: instance +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/values.yaml new file mode 100644 index 0000000000..04569505d6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/values.yaml @@ -0,0 +1,366 @@ +# Default values for prometheus-windows-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-windows-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: "0.25.1" + pullPolicy: IfNotPresent + digest: "" + +config: |- + collectors: + enabled: '[defaults],tcp,memory,container' + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +global: + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + cattle: + systemDefaultRegistry: "" + +service: + type: ClusterIP + port: 9796 + nodePort: + portName: windows-metrics + annotations: {} + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: true + additionalLabels: {} + namespace: "" + + jobLabel: "component" + + # List of pod labels to add to windows exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: ["component"] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Windows Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m +# memory: 30Mi + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + +rbac: + ## If true, create & use RBAC resources + ## + create: true + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to windows exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to windows exporter pods +podLabels: {} + +# Annotations to be added to windows exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-windows-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: windows + # kubernetes.io/arch: amd64 + +tolerations: + - effect: NoSchedule + operator: Exists + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.service.services-where +# - "Name LIKE 'sql%'" + +## Additional mounts from the host to windows-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file +## +sidecars: [] +## - name: nvidia-dcgm-exporter +## image: nvidia/dcgm-exporter:1.4.3 + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +## - name: collector-textfiles +## mountPath: /run/prometheus +## readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/nginx.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/nginx.json new file mode 100644 index 0000000000..565352235a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/nginx.json @@ -0,0 +1,1445 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "$datasource", + "enable": true, + "expr": "sum(changes(nginx_ingress_controller_config_last_reload_successful_timestamp_seconds{instance!=\"unknown\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[30s])) by (controller_class)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Config Reloads", + "showIn": 0, + "step": "30s", + "tagKeys": "controller_class", + "tags": [], + "titleFormat": "Config Reloaded", + "type": "tags" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1534359654832, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 82, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",state=\"active\"}[2m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Connections", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Controller Success Rate (non-4|5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 81, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(irate(nginx_ingress_controller_success{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[1m])) * 60", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Config Reloads", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(nginx_ingress_controller_config_last_reload_successful{controller_pod=~\"$controller\",controller_namespace=~\"$namespace\"} == 0)", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Last Config Failed", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "200px", + "id": 86, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "network", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Request Volume", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Success Rate (non-4|5xx responses)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 77, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) ", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "", + "id": 79, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg (rate (nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) ", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "hideTimeOverride": false, + "id": 75, + "links": [], + "pageSize": 7, + "repeat": null, + "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Ingress", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ingress", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ops" + }, + { + "alias": "Errors", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "IN", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "OUT", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "Bps" + } + ], + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "E" + }, + { + "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "F" + }, + { + "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "G" + } + ], + "timeFrom": null, + "title": "Ingress Percentile Response Times and Transfer Rates", + "transform": "table", + "transparent": false, + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "height": "1024", + "id": 85, + "links": [], + "pageSize": 7, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "TTL", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0", + "691200" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "avg(nginx_ingress_controller_ssl_expire_time_seconds{kubernetes_pod_name=~\"$controller\",namespace=~\"$namespace\",ingress=~\"$ingress\"}) by (host) - time()", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ host }}", + "metric": "gke_letsencrypt_cert_expiration", + "refId": "A", + "step": 1 + } + ], + "title": "Ingress Certificate Expiry", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller Class", + "multi": false, + "name": "controller_class", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller", + "multi": false, + "name": "controller", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller_pod=~\"$controller\"}, ingress) ", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Ingress Controller", + "uid": "nginx", + "version": 1 +} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/request-handling-performance.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/request-handling-performance.json new file mode 100644 index 0000000000..156e33123d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/request-handling-performance.json @@ -0,0 +1,963 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": 9614, + "graphTooltip": 1, + "id": null, + "iteration": 1582146566338, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total time taken for nginx and upstream servers to process a request and send a response", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total request handling time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The time spent on receiving the response from the upstream server", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 94, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum by (path)(\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, its median upstream response time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n .5,\n sum by (le, path)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median upstream response time by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Percentage of 4xx and 5xx responses among all responses.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~ \"[4-5].*\"\n}[1m])) / sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error rate by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, the sum of upstream request time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_response_duration_seconds_sum{ingress =~ \"$ingress\"}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream time consumed by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum (\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~\"[4-5].*\",\n }[1m]\n )\n ) by(path, status)\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }} {{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate (\n nginx_ingress_controller_response_size_sum {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path) / sum (\n rate(\n nginx_ingress_controller_response_size_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "D" + }, + { + "expr": " sum (rate(nginx_ingress_controller_response_size_bucket{\n ingress =~ \"$ingress\",\n }[1m])) by (le)\n", + "hide": true, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average response size by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_sum {\n ingress =~ \"$ingress\",\n }[1m]\n)) / sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "average", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream service latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 22, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(nginx_ingress_controller_requests, ingress) ", + "hide": 0, + "includeAll": true, + "label": "Service Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests, ingress) ", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Request Handling Performance", + "uid": "4GFbkOsZk", + "version": 1 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json new file mode 100644 index 0000000000..1d4943501b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json @@ -0,0 +1,793 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m] ({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m])) by (instance)", + "interval": "", + "legendFormat": "Load[5m] ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m])) by (instance)", + "interval": "", + "legendFormat": "Load[1m] ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m])) by (instance)", + "interval": "", + "legendFormat": "Load[15m] ({{instance}})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) by (instance) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes) by (instance) ", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Read ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Write ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Errors ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Errors ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Dropped ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Dropped ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster (Nodes)", + "uid": "rancher-cluster-nodes-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster.json new file mode 100644 index 0000000000..24385a237a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster.json @@ -0,0 +1,776 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval]))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes)", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster", + "uid": "rancher-cluster-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundle.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundle.json new file mode 100644 index 0000000000..698f48aeed --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundle.json @@ -0,0 +1,246 @@ +{ + "description": "Bundle", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Bundle", + "uid": "fleet-bundle" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundledeployment.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundledeployment.json new file mode 100644 index 0000000000..c81f7a6212 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundledeployment.json @@ -0,0 +1,219 @@ +{ + "description": "BundleDeployment", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"}) / sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\"})" + } + ], + "title": "Ready BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundledeployment_state, cluster_namespace)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / BundleDeployment", + "uid": "fleet-bundledeployment" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/cluster.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/cluster.json new file mode 100644 index 0000000000..73bdea4834 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/cluster.json @@ -0,0 +1,484 @@ +{ + "description": "Cluster", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"}) / sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_desired_ready_git_repos, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_desired_ready_git_repos{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Cluster", + "uid": "fleet-cluster" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/clustergroup.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/clustergroup.json new file mode 100644 index 0000000000..ce3df87b21 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/clustergroup.json @@ -0,0 +1,468 @@ +{ + "description": "ClusterGroup", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"}) - sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})) / sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_group_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_group_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / ClusterGroup", + "uid": "fleet-cluster-group" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/controller-runtime.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/controller-runtime.json new file mode 100644 index 0000000000..23a81f2a8c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/controller-runtime.json @@ -0,0 +1,454 @@ +{ + "description": "Controller Runtime", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "controller_runtime_active_workers{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{controller}} {{instance}}" + } + ], + "title": "Number of Workers in Use", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Reconciliation Error Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Total Reconciliation Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "workqueue_depth{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "WorkQueue Depth", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P50", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P90", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P99", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_adds_total{job=\"$job\", namespace=\"$namespace\"}[2m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Add Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(workqueue_unfinished_work_seconds{job=\"$job\", namespace=\"$namespace\"}[5m])", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Unfinished Seconds", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 10, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 50th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 11, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 90th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 12, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 99th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 96 + }, + "id": 13, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_retries_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Retries Rate", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(controller_runtime_reconcile_total, namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "job", + "query": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"}, job)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Controller-Runtime", + "uid": "fleet-controller-runtime" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/gitrepo.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/gitrepo.json new file mode 100644 index 0000000000..1a50c2937d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/gitrepo.json @@ -0,0 +1,325 @@ +{ + "description": "GitRepo", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_gitrepo_desired_ready_clusters, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_gitrepo_desired_ready_clusters{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / GitRepo", + "uid": "fleet-gitrepo" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/home/rancher-default-home.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/home/rancher-default-home.json new file mode 100644 index 0000000000..3fce207561 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/home/rancher-default-home.json @@ -0,0 +1,1290 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "", + "type": "welcome" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[5m])))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "CPU Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 4 + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"})) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Memory Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 4 + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Disk Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 9 + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode!=\"idle\"}[5m]))", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 9 + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_cpu_cores{}) OR sum(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 9 + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 9 + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_memory_bytes{}) OR sum(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 9 + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) - sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 2051, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", mode=\"idle\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 2052, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "100 * (1- sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) by (instance) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) by (instance))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 2053, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(1 - ((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"} OR on() vector(0)))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0)))) * 100", + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "(1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "B" + }, + { + "expr": "(1 - (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) / sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "folderId": 0, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 18 + }, + "headings": true, + "id": 3, + "limit": 30, + "links": [], + "query": "", + "recent": true, + "search": true, + "starred": false, + "tags": [], + "title": "Dashboards", + "type": "dashlist" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 2055, + "options": { + "content": "## About Rancher Monitoring\n\nRancher Monitoring is a Helm chart developed by Rancher that is powered by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). It is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm chart maintained by the Prometheus community.\n\nBy default, the chart deploys Grafana alongside a set of Grafana dashboards curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project.\n\nFor more information on how Rancher Monitoring differs from [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack), please view the CHANGELOG.md of the rancher-monitoring chart located in the [rancher/charts](https://github.com/rancher/charts) repository.\n\nFor more information about how to configure Rancher Monitoring, please view the [Rancher docs](https://rancher.com/docs/rancher/v2.x/en/).\n\n", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "text" + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Home", + "uid": "rancher-home-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json new file mode 100644 index 0000000000..8af4b81ce0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json @@ -0,0 +1,687 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 32, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic In ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic Out ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes) by (instance)", + "interval": "", + "legendFormat": "DB Size ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Watch Streams ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Lease Watch Stream ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Committed ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Applied ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Failed ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending) by (instance)", + "interval": "", + "legendFormat": "Proposal Pending ({{instance}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Rate ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Failure Rate ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync ({{instance}})", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd (Nodes)", + "uid": "rancher-etcd-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd.json new file mode 100644 index 0000000000..0c058cafb9 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd.json @@ -0,0 +1,669 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 33, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic In", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic Out", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes)", + "interval": "", + "legendFormat": "DB Size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Watch Streams", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Lease Watch Stream", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Committed", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Applied", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Failed", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending)", + "interval": "", + "legendFormat": "Proposal Pending", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Rate", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Failure Rate", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd", + "uid": "rancher-etcd-1", + "version": 4 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json new file mode 100644 index 0000000000..b31358eaaf --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json @@ -0,0 +1,527 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 30, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (instance, code)", + "interval": "", + "legendFormat": "{{code}}({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (instance, name)", + "interval": "", + "legendFormat": "Deployment Depth ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (instance, name)", + "interval": "", + "legendFormat": "Volumes Depth ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicaSet Depth ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (instance, name)", + "interval": "", + "legendFormat": "Service Depth ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (instance, name)", + "interval": "", + "legendFormat": "ServiceAccount Depth ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (instance, name)", + "interval": "", + "legendFormat": "Endpoint Depth ({{instance}})", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (instance, name)", + "interval": "", + "legendFormat": "DaemonSet Depth ({{instance}})", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (instance, name)", + "interval": "", + "legendFormat": "StatefulSet Depth ({{instance}})", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicationManager Depth ({{instance}})", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"}) by (instance)", + "interval": "", + "legendFormat": "Reading ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"}) by (instance)", + "interval": "", + "legendFormat": "Waiting ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"}) by (instance)", + "interval": "", + "legendFormat": "Writing ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Accepted ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Handled ({{instance}})", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components (Nodes)", + "uid": "rancher-k8s-components-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components.json new file mode 100644 index 0000000000..44cf97f9fd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components.json @@ -0,0 +1,519 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 31, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (code)", + "interval": "", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (name)", + "interval": "", + "legendFormat": "Deployment Depth", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (name)", + "interval": "", + "legendFormat": "Volumes Depth", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (name)", + "interval": "", + "legendFormat": "Replicaset Depth", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (name)", + "interval": "", + "legendFormat": "Service Depth", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (name)", + "interval": "", + "legendFormat": "ServiceAccount Depth", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (name)", + "interval": "", + "legendFormat": "Endpoint Depth", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (name)", + "interval": "", + "legendFormat": "DaemonSet Depth", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (name)", + "interval": "", + "legendFormat": "StatefulSet Depth", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (name)", + "interval": "", + "legendFormat": "ReplicationManager Depth", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"})", + "interval": "", + "legendFormat": "Reading", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"})", + "interval": "", + "legendFormat": "Waiting", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"})", + "interval": "", + "legendFormat": "Writing", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Accepted", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Handled", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components", + "uid": "rancher-k8s-components-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node-detail.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node-detail.json new file mode 100644 index 0000000000..920fb94cf7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node-detail.json @@ -0,0 +1,805 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{mode}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\"}[$__rate_interval])) by (mode)", + "interval": "", + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / (node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device))", + "interval": "", + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Read ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Write ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Errors ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Errors ({{device}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Dropped ({{device}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Dropped ({{device}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node (Detail)", + "uid": "rancher-node-detail-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node.json new file mode 100644 index 0000000000..367df3cc9d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node.json @@ -0,0 +1,792 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\", mode=\"idle\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / sum(node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node", + "uid": "rancher-node-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/performance/performance-debugging.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/performance/performance-debugging.json new file mode 100644 index 0000000000..454bc39390 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/performance/performance-debugging.json @@ -0,0 +1,1652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_sum[5m]))\n/\nsum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_count[5m])))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Average Execution Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1390", + "format": "short", + "label": "Execution Time in Seconds", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1391", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(steve_api_request_time_sum{resource!=\"subscribe\"}[5m]))\n/\nsum by (resource, method, code) (rate(steve_api_request_time_count{resource!=\"subscribe\"}[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Rancher API Average Request Times Over Last 5 Minutes (Top 20) (Subscribes Omitted)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:178", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:179", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "rate(steve_api_request_time_sum{resource=\"subscribe\"}[5m])\n/\nrate(steve_api_request_time_count{resource=\"subscribe\"}[5m])", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Subscribe Average Request Times Over Last 5 Minutes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:368", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:369", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,workqueue_depth)", + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Controller Work Queue Depth (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1553", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1554", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 16, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method, code) (steve_api_total_requests))", + "instant": false, + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Rancher Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:290", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:291", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method) (steve_api_total_requests{code!=\"200\",code!=\"201\"}))", + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Failed Rancher API Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:428", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:429", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_store_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_store_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Store Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:662", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:663", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 62 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_client_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_client_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Client Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1710", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1711", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,lasso_controller_total_cached_object)", + "interval": "", + "legendFormat": "{{kind}} {{version}} {{group}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cached Objects by GroupVersionKind (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:744", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:745", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 78 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Handler Executions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:824", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:825", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 86 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20, sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution{has_error=\"true\"}\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Handler Executions with Error (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1230", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1231", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 102 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution{has_error=\"true\"}[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 110 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,session_server_total_transmit_bytes)", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Data Transmitted by Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1953", + "format": "decbytes", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1954", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 118 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "session_server_total_transmit_error_bytes", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Errors for Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2045", + "format": "ms", + "label": "Error Data", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2046", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 126 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "session_server_total_add_websocket_session - (session_server_total_remove_websocket_session or (0 * session_server_total_add_websocket_session))", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Active Connections (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 134 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_remove_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Removed Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 142 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_add_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Added Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2117", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2118", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rancher Performance Debugging", + "uid": "tfrfU0a7k", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod-containers.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod-containers.json new file mode 100644 index 0000000000..cf78a2204c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod-containers.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "CFS throttled ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "System ({{container}})", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Total ({{container}})", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "User ({{container}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}) by (container)", + "interval": "", + "legendFormat": "({{container}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Dropped ({{container}})", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Errors ({{container}})", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Errors ({{container}})", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Dropped ({{container}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Write ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Read ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod (Containers)", + "uid": "rancher-pod-containers-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod.json new file mode 100644 index 0000000000..4859eccc74 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod", + "uid": "rancher-pod-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload-pods.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload-pods.json new file mode 100644 index 0000000000..92c0d24a6e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload-pods.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "CFS throttled ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "System ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Total ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "User ({{pod}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Dropped ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Errors ({{pod}})", + "refId": "D" + }, + { + "expr": "(sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Errors ({{pod}})", + "refId": "E" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Dropped ({{pod}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Write ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Read ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload (Pods)", + "uid": "rancher-workload-pods-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload.json new file mode 100644 index 0000000000..9f5317c2f0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum((sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum((sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(container_memory_working_set_bytes{namespace=~\"$namespace\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum((sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum((sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum((sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload", + "uid": "rancher-workload-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh new file mode 100644 index 0000000000..89431e7132 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e +set -x + +# node-exporter +kubectl delete daemonset -l app=prometheus-node-exporter,release=rancher-monitoring --ignore-not-found=true + +# prometheus-adapter +kubectl delete deployments -l app=prometheus-adapter,release=rancher-monitoring --ignore-not-found=true + +# kube-state-metrics +kubectl delete deployments -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true +kubectl delete statefulsets -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/NOTES.txt new file mode 100644 index 0000000000..371f3ae398 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "kube-prometheus-stack.namespace" . }} get pods -l "release={{ $.Release.Name }}" + +Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl new file mode 100644 index 0000000000..d2207dd90d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl @@ -0,0 +1,459 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +{{/* +https://github.com/helm/helm/issues/4535#issuecomment-477778391 +Usage: {{ include "call-nested" (list . "SUBCHART_NAME" "TEMPLATE") }} +e.g. {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +*/}} +{{- define "call-nested" }} +{{- $dot := index . 0 }} +{{- $subchart := index . 1 | splitList "." }} +{{- $template := index . 2 }} +{{- $values := $dot.Values }} +{{- range $subchart }} +{{- $values = index $values . }} +{{- end }} +{{- include $template (dict "Chart" (dict "Name" (last $subchart)) "Values" $values "Release" $dot.Release "Capabilities" $dot.Capabilities) }} +{{- end }} + +# Special Exporters +{{- define "exporter.kubeEtcd.enabled" -}} +{{- if or .Values.kubeEtcd.enabled .Values.rkeEtcd.enabled .Values.kubeAdmEtcd.enabled .Values.rke2Etcd.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.enabled" -}} +{{- if or .Values.kubeControllerManager.enabled .Values.rkeControllerManager.enabled .Values.k3sServer.enabled .Values.kubeAdmControllerManager.enabled .Values.rke2ControllerManager.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.enabled" -}} +{{- if or .Values.kubeScheduler.enabled .Values.rkeScheduler.enabled .Values.k3sServer.enabled .Values.kubeAdmScheduler.enabled .Values.rke2Scheduler.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.enabled" -}} +{{- if or .Values.kubeProxy.enabled .Values.rkeProxy.enabled .Values.k3sServer.enabled .Values.kubeAdmProxy.enabled .Values.rke2Proxy.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.enabled" -}} +{{- if or .Values.kubelet.enabled .Values.hardenedKubelet.enabled .Values.k3sServer.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-controller-manager +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-scheduler +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-proxy +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kubelet +{{- end -}} +{{- end }} + +{{- define "kubelet.serviceMonitor.resourcePath" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if not (eq .Values.kubelet.serviceMonitor.resourcePath "/metrics/resource/v1alpha1") -}} +{{ .Values.kubelet.serviceMonitor.resourcePath }} +{{- else if semverCompare ">=1.20.0-0" $kubeTargetVersion -}} +/metrics/resource +{{- else -}} +/metrics/resource/v1alpha1 +{{- end -}} +{{- end }} + +{{- define "rancher.serviceMonitor.selector" -}} +{{- if .Values.rancherMonitoring.selector }} +{{ .Values.rancherMonitoring.selector | toYaml }} +{{- else }} +{{- $rancherDeployment := (lookup "apps/v1" "Deployment" "cattle-system" "rancher") }} +{{- if $rancherDeployment }} +matchLabels: + app: rancher + chart: {{ index $rancherDeployment.metadata.labels "chart" }} + release: rancher +{{- end }} +{{- end }} +{{- end }} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Prometheus Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "kube-prometheus-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "kube-prometheus-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 26 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Fullname suffixed with -operator */}} +{{/* Adding 9 to 26 truncation of kube-prometheus-stack.fullname */}} +{{- define "kube-prometheus-stack.operator.fullname" -}} +{{- if .Values.prometheusOperator.fullnameOverride -}} +{{- .Values.prometheusOperator.fullnameOverride | trunc 35 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-operator" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} +{{- end }} + +{{/* Prometheus custom resource instance name */}} +{{- define "kube-prometheus-stack.prometheus.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-prometheus" }} +{{- end }} +{{- end }} + +{{/* Prometheus apiVersion for networkpolicy */}} +{{- define "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} + +{{/* Alertmanager custom resource instance name */}} +{{- define "kube-prometheus-stack.alertmanager.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-alertmanager" -}} +{{- end }} +{{- end }} + +{{/* Fullname suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.fullname" -}} +{{- printf "%s-thanos-ruler" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} + +{{/* Shortened name suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.name" -}} +{{- default (printf "%s-thanos-ruler" (include "kube-prometheus-stack.name" .)) .Values.thanosRuler.name -}} +{{- end }} + + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "kube-prometheus-stack.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "kube-prometheus-stack.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }} +chart: {{ template "kube-prometheus-stack.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.operator.fullname" .) .Values.prometheusOperator.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (printf "%s-webhook" (include "kube-prometheus-stack.operator.fullname" .)) .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of prometheus service account to use */}} +{{- define "kube-prometheus-stack.prometheus.serviceAccountName" -}} +{{- if .Values.prometheus.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-prometheus") .Values.prometheus.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of alertmanager service account to use */}} +{{- define "kube-prometheus-stack.alertmanager.serviceAccountName" -}} +{{- if .Values.alertmanager.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-alertmanager") .Values.alertmanager.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.alertmanager.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of thanosRuler service account to use */}} +{{- define "kube-prometheus-stack.thanosRuler.serviceAccountName" -}} +{{- if .Values.thanosRuler.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.thanosRuler.name" .) .Values.thanosRuler.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.thanosRuler.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the grafana namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-grafana.namespace" -}} + {{- if .Values.grafana.namespaceOverride -}} + {{- .Values.grafana.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Allow kube-state-metrics job name to be overridden +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.name" -}} + {{- if index .Values "kube-state-metrics" "nameOverride" -}} + {{- index .Values "kube-state-metrics" "nameOverride" -}} + {{- else -}} + {{- print "kube-state-metrics" -}} + {{- end -}} +{{- end -}} + +{{/* +Use the kube-state-metrics namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.namespace" -}} + {{- if index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the prometheus-node-exporter namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-prometheus-node-exporter.namespace" -}} + {{- if index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Allow KubeVersion to be overridden. */}} +{{- define "kube-prometheus-stack.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{/* Get Ingress API Version */}} +{{- define "kube-prometheus-stack.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* Check Ingress stability */}} +{{- define "kube-prometheus-stack.ingress.isStable" -}} + {{- eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* Check Ingress supports pathType */}} +{{/* pathType was added to networking.k8s.io/v1beta1 in Kubernetes 1.18 */}} +{{- define "kube-prometheus-stack.ingress.supportsPathType" -}} + {{- or (eq (include "kube-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "kube-prometheus-stack.kubeVersion" .))) -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "kube-prometheus-stack.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} + {{- end -}} + +{{/* Get value based on current Kubernetes version */}} +{{- define "kube-prometheus-stack.kubeVersionDefaultValue" -}} + {{- $values := index . 0 -}} + {{- $kubeVersion := index . 1 -}} + {{- $old := index . 2 -}} + {{- $new := index . 3 -}} + {{- $default := index . 4 -}} + {{- if kindIs "invalid" $default -}} + {{- if semverCompare $kubeVersion (include "kube-prometheus-stack.kubeVersion" $values) -}} + {{- print $new -}} + {{- else -}} + {{- print $old -}} + {{- end -}} + {{- else -}} + {{- print $default }} + {{- end -}} +{{- end -}} + +{{/* Get value for kube-controller-manager depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeControllerManager.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.22-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Get value for kube-scheduler depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeScheduler.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.23-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +To help compatibility with other charts which use global.imagePullSecrets. +Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). +global: + imagePullSecrets: + - name: pullSecret1 + - name: pullSecret2 + +or + +global: + imagePullSecrets: + - pullSecret1 + - pullSecret2 +*/}} +{{- define "kube-prometheus-stack.imagePullSecrets" -}} +{{- range .Values.global.imagePullSecrets }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{- define "kube-prometheus-stack.operator.admission-webhook.dnsNames" }} +{{- $fullname := include "kube-prometheus-stack.operator.fullname" . }} +{{- $namespace := include "kube-prometheus-stack.namespace" . }} +{{- $fullname }} +{{ $fullname }}.{{ $namespace }}.svc +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +{{ $fullname }}-webhook +{{ $fullname }}-webhook.{{ $namespace }}.svc +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/alertmanager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/alertmanager.yaml new file mode 100644 index 0000000000..19044054ac --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/alertmanager.yaml @@ -0,0 +1,191 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.annotations }} + annotations: +{{ toYaml .Values.alertmanager.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.alertmanagerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.alertmanager.alertmanagerSpec.image.registry }} + {{- if and .Values.alertmanager.alertmanagerSpec.image.tag .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}" + {{- end }} + version: {{ .Values.alertmanager.alertmanagerSpec.image.tag }} + {{- if .Values.alertmanager.alertmanagerSpec.image.sha }} + sha: {{ .Values.alertmanager.alertmanagerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.alertmanager.alertmanagerSpec.replicas }} + listenLocal: {{ .Values.alertmanager.alertmanagerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.alertmanager.alertmanagerSpec.automountServiceAccountToken }} +{{- if .Values.alertmanager.alertmanagerSpec.externalUrl }} + externalUrl: "{{ tpl .Values.alertmanager.alertmanagerSpec.externalUrl . }}" +{{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.alertmanager.ingress.hosts 0) . }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ .Values.namespaceOverride }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.nodeSelector }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.alertmanager.alertmanagerSpec.paused }} + logFormat: {{ .Values.alertmanager.alertmanagerSpec.logFormat | quote }} + logLevel: {{ .Values.alertmanager.alertmanagerSpec.logLevel | quote }} + retention: {{ .Values.alertmanager.alertmanagerSpec.retention | quote }} +{{- if .Values.alertmanager.alertmanagerSpec.secrets }} + secrets: +{{ toYaml .Values.alertmanager.alertmanagerSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configSecret }} + configSecret: {{ .Values.alertmanager.alertmanagerSpec.configSecret }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configMaps }} + configMaps: +{{ toYaml .Values.alertmanager.alertmanagerSpec.configMaps | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector }} + alertmanagerConfigSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector | indent 4) . }} +{{ else }} + alertmanagerConfigSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector }} + alertmanagerConfigNamespaceSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector | indent 4) . }} +{{ else }} + alertmanagerConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.web }} + web: +{{ toYaml .Values.alertmanager.alertmanagerSpec.web | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration }} + alertmanagerConfiguration: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy }} + alertmanagerConfigMatcherStrategy: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.resources }} + resources: +{{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + routePrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.alertmanagerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.storage }} + storage: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.storage | indent 4) . }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.alertmanager.alertmanagerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.alertmanager.alertmanagerSpec.podAntiAffinity .Values.alertmanager.alertmanagerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.affinity }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.tolerations }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.containers }} + containers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.initContainers }} + initContainers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.priorityClassName }} + priorityClassName: {{.Values.alertmanager.alertmanagerSpec.priorityClassName }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.additionalPeers }} + additionalPeers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.additionalPeers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumes }} + volumes: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.alertmanager.alertmanagerSpec.portName }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} + clusterAdvertiseAddress: {{ .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} + clusterGossipInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} + clusterPeerTimeout: {{ .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} + clusterPushpullInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} + forceEnableClusterMode: {{ .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.minReadySeconds }} + minReadySeconds: {{ .Values.alertmanager.alertmanagerSpec.minReadySeconds }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/extrasecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/extrasecret.yaml new file mode 100644 index 0000000000..ecd8f47021 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.alertmanager.extraSecret.data -}} +{{- $secretName := printf "alertmanager-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.alertmanager.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.extraSecret.annotations }} + annotations: +{{ toYaml .Values.alertmanager.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.alertmanager.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingress.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingress.yaml new file mode 100644 index 0000000000..be9f5aa279 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingress.yaml @@ -0,0 +1,78 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} +{{- $pathType := .Values.alertmanager.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $backendServiceName := .Values.alertmanager.ingress.serviceName | default (printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager") }} +{{- $servicePort := .Values.alertmanager.ingress.servicePort | default .Values.alertmanager.service.port -}} +{{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} +{{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.alertmanager.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.alertmanager.ingress.labels }} +{{ toYaml .Values.alertmanager.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.alertmanager.ingress.ingressClassName }} + ingressClassName: {{ .Values.alertmanager.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.alertmanager.ingress.hosts }} + {{- range $host := .Values.alertmanager.ingress.hosts }} + - host: {{ tpl $host $ | quote }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.alertmanager.ingress.tls }} + tls: +{{ tpl (toYaml .Values.alertmanager.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingressperreplica.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingressperreplica.yaml new file mode 100644 index 0000000000..b2e00a4162 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled .Values.alertmanager.ingressPerReplica.enabled }} +{{- $pathType := .Values.alertmanager.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $servicePort := .Values.alertmanager.service.port -}} +{{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml new file mode 100644 index 0000000000..b183403125 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.alertmanager.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.alertmanager.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-role.yaml new file mode 100644 index 0000000000..8810e93ded --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-rolebinding.yaml new file mode 100644 index 0000000000..794f4ad178 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp.yaml new file mode 100644 index 0000000000..07b616b5cb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/secret.yaml new file mode 100644 index 0000000000..d2fe84a7bf --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/secret.yaml @@ -0,0 +1,35 @@ +{{- if and (.Values.alertmanager.enabled) (not .Values.alertmanager.alertmanagerSpec.useExistingSecret) }} +{{/* This file is applied when the operation is helm install and the target secret does not exist. */}} +{{- $secretName := (printf "alertmanager-%s" (include "kube-prometheus-stack.alertmanager.crname" .)) }} +{{- if or (not (lookup "v1" "Secret" (include "kube-prometheus-stack.namespace" .) $secretName)) (eq .Values.alertmanager.secret.recreateIfExists true) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/resource-policy": keep +{{- if .Values.alertmanager.secret.annotations }} +{{ toYaml .Values.alertmanager.secret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if .Values.alertmanager.tplConfig }} +{{- if .Values.alertmanager.stringConfig }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.stringConfig) . | b64enc | quote }} +{{- else if eq (typeOf .Values.alertmanager.config) "string" }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.config) . | b64enc | quote }} +{{- else }} + alertmanager.yaml: {{ tpl (toYaml .Values.alertmanager.config) . | b64enc | quote }} +{{- end }} +{{- else }} + alertmanager.yaml: {{ toYaml .Values.alertmanager.config | b64enc | quote }} +{{- end }} +{{- range $key, $val := .Values.alertmanager.templateFiles }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/service.yaml new file mode 100644 index 0000000000..373de328a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/service.yaml @@ -0,0 +1,68 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.alertmanager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + self-monitor: {{ .Values.alertmanager.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.service.labels }} +{{ toYaml .Values.alertmanager.service.labels | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.annotations }} + annotations: +{{ toYaml .Values.alertmanager.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.service.clusterIP }} + clusterIP: {{ .Values.alertmanager.service.clusterIP }} +{{- end }} +{{- if .Values.alertmanager.service.externalIPs }} + externalIPs: +{{ toYaml .Values.alertmanager.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.alertmanager.service.loadBalancerIP }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.alertmanager.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.alertmanager.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq .Values.alertmanager.service.type "NodePort" }} + nodePort: {{ .Values.alertmanager.service.nodePort }} + {{- end }} + port: {{ .Values.alertmanager.service.port }} + targetPort: {{ .Values.alertmanager.service.targetPort }} + protocol: TCP + - name: reloader-web + {{- if semverCompare ">=1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: 8080 + targetPort: reloader-web +{{- if .Values.alertmanager.service.additionalPorts }} +{{ toYaml .Values.alertmanager.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- if .Values.alertmanager.service.sessionAffinity }} + sessionAffinity: {{ .Values.alertmanager.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.alertmanager.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.alertmanager.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.alertmanager.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceaccount.yaml new file mode 100644 index 0000000000..745ced8bde --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.alertmanager.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.alertmanager.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2}} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/servicemonitor.yaml new file mode 100644 index 0000000000..6233690019 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/servicemonitor.yaml @@ -0,0 +1,84 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.alertmanager.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.alertmanager.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + enableHttp2: {{ .Values.alertmanager.serviceMonitor.enableHttp2 }} + {{- if .Values.alertmanager.serviceMonitor.interval }} + interval: {{ .Values.alertmanager.serviceMonitor.interval }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.alertmanager.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.scheme }} + scheme: {{ .Values.alertmanager.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.alertmanager.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.alertmanager.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.alertmanager.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.alertmanager.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.alertmanager.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.alertmanager.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.alertmanager.serviceMonitor.interval .interval }} + interval: {{ default $.Values.alertmanager.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.alertmanager.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.alertmanager.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceperreplica.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceperreplica.yaml new file mode 100644 index 0000000000..75a13bdf97 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceperreplica.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $serviceValues := .Values.alertmanager.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" $ }} + statefulset.kubernetes.io/pod-name: alertmanager-{{ include "kube-prometheus-stack.alertmanager.crname" $ }}-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/service.yaml new file mode 100644 index 0000000000..b8618f7558 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/service.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + jobLabel: coredns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.coreDns.serviceMonitor.port }} + port: {{ .Values.coreDns.service.port }} + protocol: TCP + targetPort: {{ .Values.coreDns.service.targetPort }} + selector: + {{- if .Values.coreDns.service.selector }} +{{ toYaml .Values.coreDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml new file mode 100644 index 0000000000..dc15a06937 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + {{- with .Values.coreDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.coreDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.coreDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.coreDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.coreDns.serviceMonitor.port }} + {{- if .Values.coreDns.serviceMonitor.interval}} + interval: {{ .Values.coreDns.serviceMonitor.interval }} + {{- end }} + {{- if .Values.coreDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.coreDns.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.coreDns.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.coreDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.coreDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml new file mode 100644 index 0000000000..66e777632e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.kubeApiServer.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-apiserver + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: default + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-apiserver + {{- with .Values.kubeApiServer.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubeApiServer.serviceMonitor | nindent 2 }} + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeApiServer.serviceMonitor.interval }} + interval: {{ .Values.kubeApiServer.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubeApiServer.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeApiServer.serviceMonitor.proxyUrl }} + {{- end }} + port: https + scheme: https + metricRelabelings: + {{- if .Values.kubeApiServer.serviceMonitor.metricRelabelings }} +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeApiServer.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.relabelings | indent 6) . }} +{{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + serverName: {{ .Values.kubeApiServer.tlsConfig.serverName }} + insecureSkipVerify: {{ .Values.kubeApiServer.tlsConfig.insecureSkipVerify }} + jobLabel: {{ .Values.kubeApiServer.serviceMonitor.jobLabel }} + namespaceSelector: + matchNames: + - default + selector: +{{ toYaml .Values.kubeApiServer.serviceMonitor.selector | indent 4 }} +{{- end}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml new file mode 100644 index 0000000000..6a6afa6412 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + k8s-app: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeControllerManager.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/service.yaml new file mode 100644 index 0000000000..43b1a976d5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + jobLabel: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.targetPort) }} +{{- if .Values.kubeControllerManager.endpoints }}{{- else }} + selector: + {{- if .Values.kubeControllerManager.service.selector }} +{{ toYaml .Values.kubeControllerManager.service.selector | indent 4 }} + {{- else}} + component: kube-controller-manager + {{- end}} +{{- end }} + type: ClusterIP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml new file mode 100644 index 0000000000..7ed3baa65f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + {{- with .Values.kubeControllerManager.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeControllerManager.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeControllerManager.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeControllerManager.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- if .Values.kubeControllerManager.serviceMonitor.interval }} + interval: {{ .Values.kubeControllerManager.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeControllerManager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeControllerManager.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . false true .Values.kubeControllerManager.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . nil true .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeControllerManager.serviceMonitor.serverName }} + serverName: {{ .Values.kubeControllerManager.serviceMonitor.serverName }} + {{- end }} + {{- end }} + metricRelabelings: + {{- if.Values.kubeControllerManager.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeControllerManager.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/service.yaml new file mode 100644 index 0000000000..81b2c9930c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/service.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + jobLabel: kube-dns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics-dnsmasq + port: {{ .Values.kubeDns.service.dnsmasq.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.dnsmasq.targetPort }} + - name: http-metrics-skydns + port: {{ .Values.kubeDns.service.skydns.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.skydns.targetPort }} + selector: + {{- if .Values.kubeDns.service.selector }} +{{ toYaml .Values.kubeDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml new file mode 100644 index 0000000000..9fa41b575f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + {{- with .Values.kubeDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics-dnsmasq + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeDns.serviceMonitor.proxyUrl}} + {{- end }} + metricRelabelings: + {{- if .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeDns.serviceMonitor.dnsmasqRelabelings }} + relabelings: +{{ toYaml .Values.kubeDns.serviceMonitor.dnsmasqRelabelings | indent 4 }} +{{- end }} + - port: http-metrics-skydns + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubeDns.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml new file mode 100644 index 0000000000..e366447577 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + k8s-app: etcd-server +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeEtcd.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/service.yaml new file mode 100644 index 0000000000..d07d4f35e3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + jobLabel: kube-etcd +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeEtcd.service.targetPort }} +{{- if .Values.kubeEtcd.endpoints }}{{- else }} + selector: + {{- if .Values.kubeEtcd.service.selector }} +{{ toYaml .Values.kubeEtcd.service.selector | indent 4 }} + {{- else}} + component: etcd + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml new file mode 100644 index 0000000000..26fdbdbed3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + {{- with .Values.kubeEtcd.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeEtcd.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeEtcd.serviceMonitor | nindent 4 }} + selector: + {{- if .Values.kubeEtcd.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeEtcd.serviceMonitor.port }} + {{- if .Values.kubeEtcd.serviceMonitor.interval }} + interval: {{ .Values.kubeEtcd.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeEtcd.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeEtcd.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq .Values.kubeEtcd.serviceMonitor.scheme "https" }} + scheme: https + tlsConfig: + {{- if .Values.kubeEtcd.serviceMonitor.serverName }} + serverName: {{ .Values.kubeEtcd.serviceMonitor.serverName }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.caFile }} + caFile: {{ .Values.kubeEtcd.serviceMonitor.caFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.certFile }} + certFile: {{ .Values.kubeEtcd.serviceMonitor.certFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.keyFile }} + keyFile: {{ .Values.kubeEtcd.serviceMonitor.keyFile }} + {{- end}} + insecureSkipVerify: {{ .Values.kubeEtcd.serviceMonitor.insecureSkipVerify }} + {{- end }} + metricRelabelings: + {{- if .Values.kubeEtcd.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeEtcd.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml new file mode 100644 index 0000000000..8613e62425 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + k8s-app: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeProxy.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/service.yaml new file mode 100644 index 0000000000..8ccb2210d7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + jobLabel: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeProxy.service.targetPort }} +{{- if .Values.kubeProxy.endpoints }}{{- else }} + selector: + {{- if .Values.kubeProxy.service.selector }} +{{ toYaml .Values.kubeProxy.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-proxy + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml new file mode 100644 index 0000000000..24b0ab2001 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + {{- with .Values.kubeProxy.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeProxy.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeProxy.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeProxy.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeProxy.serviceMonitor.port }} + {{- if .Values.kubeProxy.serviceMonitor.interval }} + interval: {{ .Values.kubeProxy.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeProxy.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeProxy.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.kubeProxy.serviceMonitor.https }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- end}} + metricRelabelings: + {{- if .Values.kubeProxy.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeProxy.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeProxy.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml new file mode 100644 index 0000000000..6236b42f10 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + k8s-app: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeScheduler.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/service.yaml new file mode 100644 index 0000000000..90b3a800a4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + jobLabel: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.targetPort) }} +{{- if .Values.kubeScheduler.endpoints }}{{- else }} + selector: + {{- if .Values.kubeScheduler.service.selector }} +{{ toYaml .Values.kubeScheduler.service.selector | indent 4 }} + {{- else}} + component: kube-scheduler + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml new file mode 100644 index 0000000000..b17c4f1d47 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + {{- with .Values.kubeScheduler.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeScheduler.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeScheduler.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeScheduler.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- if .Values.kubeScheduler.serviceMonitor.interval }} + interval: {{ .Values.kubeScheduler.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeScheduler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeScheduler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . false true .Values.kubeScheduler.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . nil true .Values.kubeScheduler.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeScheduler.serviceMonitor.serverName }} + serverName: {{ .Values.kubeScheduler.serviceMonitor.serverName }} + {{- end}} + {{- end}} + metricRelabelings: + {{- if .Values.kubeScheduler.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeScheduler.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml new file mode 100644 index 0000000000..9211b3d771 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml @@ -0,0 +1,7 @@ +{{- if .Values.kubeStateMetrics.enabled }} +{{- if not (kindIs "invalid" .Values.kubeStateMetrics.serviceMonitor) }} +{{- if .Values.kubeStateMetrics.serviceMonitor.namespaceOverride }} +{{- fail "kubeStateMetrics.serviceMonitor.namespaceOverride was removed. Please use kube-state-metrics.namespaceOverride instead." }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml new file mode 100644 index 0000000000..f570fbfdbc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml @@ -0,0 +1,246 @@ +{{- if (and (not .Values.kubelet.enabled) .Values.hardenedKubelet.enabled) }} +{{ required "Cannot set .Values.hardenedKubelet.enabled=true when .Values.kubelet.enabled=false" "" }} +{{- end }} +{{- if (and .Values.kubelet.enabled .Values.kubernetesServiceMonitors.enabled (not .Values.hardenedKubelet.enabled) (not .Values.k3sServer.enabled)) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kubelet + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: {{ .Values.kubelet.namespace }} + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kubelet + {{- with .Values.kubelet.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubelet.serviceMonitor | nindent 2 }} + {{- with .Values.kubelet.serviceMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + {{- if .Values.kubelet.serviceMonitor.https }} + - port: https-metrics + scheme: https + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + metricRelabelings: + {{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: https-metrics + scheme: https + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: https-metrics + scheme: https + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: https-metrics + scheme: https + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} + {{- else }} + - port: http-metrics + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: http-metrics + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: http-metrics + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: http-metrics + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- end }} + {{- end }} + jobLabel: k8s-app + namespaceSelector: + matchNames: + - {{ .Values.kubelet.namespace }} + selector: + matchLabels: + app.kubernetes.io/name: kubelet + k8s-app: kubelet +{{- end}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/node-exporter/validate.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/node-exporter/validate.yaml new file mode 100644 index 0000000000..bdc73d6165 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/node-exporter/validate.yaml @@ -0,0 +1,3 @@ +{{- if (and (not .Values.nodeExporter.enabled) .Values.hardenedNodeExporter.enabled) }} +{{ required "Cannot set .Values.hardenedNodeExporter.enabled=true when .Values.nodeExporter.enabled=false" "" }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/extra-objects.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/extra-objects.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/extra-objects.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmap-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmap-dashboards.yaml new file mode 100644 index 0000000000..e719009ffe --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmap-dashboards.yaml @@ -0,0 +1,24 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled) .Values.grafana.forceDeployDashboards }} +{{- $files := .Files.Glob "dashboards-1.14/*.json" }} +{{- if $files }} +apiVersion: v1 +kind: ConfigMapList +items: +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 6 }} + data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmaps-datasources.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmaps-datasources.yaml new file mode 100644 index 0000000000..718020d4f6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmaps-datasources.yaml @@ -0,0 +1,81 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled) .Values.grafana.forceDeployDatasources }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-grafana-datasource + namespace: {{ default .Values.grafana.sidecar.datasources.searchNamespace (include "kube-prometheus-stack.namespace" .) }} +{{- if .Values.grafana.sidecar.datasources.annotations }} + annotations: + {{- toYaml .Values.grafana.sidecar.datasources.annotations | nindent 4 }} +{{- end }} + labels: + {{ $.Values.grafana.sidecar.datasources.label }}: {{ $.Values.grafana.sidecar.datasources.labelValue | quote }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + datasource.yaml: |- + apiVersion: 1 +{{- if .Values.grafana.deleteDatasources }} + deleteDatasources: +{{ tpl (toYaml .Values.grafana.deleteDatasources | indent 6) . }} +{{- end }} + datasources: +{{- $scrapeInterval := .Values.grafana.sidecar.datasources.defaultDatasourceScrapeInterval | default .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} +{{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} + - name: Prometheus + type: prometheus + uid: {{ .Values.grafana.sidecar.datasources.uid }} + {{- if .Values.grafana.sidecar.datasources.url }} + url: {{ .Values.grafana.sidecar.datasources.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + {{- end }} + access: proxy + isDefault: {{ .Values.grafana.sidecar.datasources.isDefaultDatasource }} + jsonData: + httpMethod: {{ .Values.grafana.sidecar.datasources.httpMethod }} + timeInterval: {{ $scrapeInterval }} + {{- if .Values.grafana.sidecar.datasources.timeout }} + timeout: {{ .Values.grafana.sidecar.datasources.timeout }} + {{- end }} +{{- if .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.createPrometheusReplicasDatasources }} +{{- range until (int .Values.prometheus.prometheusSpec.replicas) }} + - name: Prometheus-{{ . }} + type: prometheus + uid: {{ $.Values.grafana.sidecar.datasources.uid }}-replica-{{ . }} + url: http://prometheus-{{ template "kube-prometheus-stack.prometheus.crname" $ }}-{{ . }}.prometheus-operated:9090/{{ trimPrefix "/" $.Values.prometheus.prometheusSpec.routePrefix }} + access: proxy + isDefault: false + jsonData: + timeInterval: {{ $scrapeInterval }} +{{- if $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.alertmanager.enabled }} + - name: Alertmanager + type: alertmanager + uid: {{ .Values.grafana.sidecar.datasources.alertmanager.uid }} + {{- if .Values.grafana.sidecar.datasources.alertmanager.url }} + url: {{ .Values.grafana.sidecar.datasources.alertmanager.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }}/{{ trimPrefix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }} + {{- end }} + access: proxy + jsonData: + handleGrafanaManagedAlerts: {{ .Values.grafana.sidecar.datasources.alertmanager.handleGrafanaManagedAlerts }} + implementation: {{ .Values.grafana.sidecar.datasources.alertmanager.implementation }} +{{- end }} +{{- end }} +{{- if .Values.grafana.additionalDataSources }} +{{ tpl (toYaml .Values.grafana.additionalDataSources | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml new file mode 100644 index 0000000000..dfc26d7ecd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml @@ -0,0 +1,616 @@ +{{- /* +Generated from 'alertmanager-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "alertmanager-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + alertmanager-overview.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "current set of alerts stored in the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid alerts received by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Received", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Invalid", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts receive rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alerts", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Failed", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notifications Send Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "latency of notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} 99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Median", + "refId": "B" + }, + { + "expr": "sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notification Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Notifications", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "alertmanager-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "service", + "multi": false, + "name": "service", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, service)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "integration", + "options": [ + + ], + "query": "label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Alertmanager / Overview", + "uid": "alertmanager-overview", + "version": 0 + } +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml new file mode 100644 index 0000000000..bd1048b567 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml @@ -0,0 +1,1772 @@ +{{- /* +Generated from 'apiserver' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeApiServer.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "apiserver" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + apiserver.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "datasource": null, + "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "mode": "markdown", + "span": 12, + "title": "Notice", + "type": "text" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Availability (30d) > 99.000%", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "description": "How much error budget is left looking at our 0.990% availability guarantees?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "errorbudget", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ErrorBudget (30d) > 99.000%", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Read Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many read requests (LIST,GET) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Write Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / API server", + "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml new file mode 100644 index 0000000000..f4be0bbd45 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -0,0 +1,1882 @@ +{{- /* +Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "cluster-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + cluster-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "namespace", + "value": "namespace" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth History", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "What is TCP Retransmit?", + "url": "https://accedian.com/enterprises/blog/network-packet-loss-retransmissions-and-duplicate-acknowledgements/" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP Retransmits out of all sent segments", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Why monitor SYN retransmits?", + "url": "https://github.com/prometheus/node_exporter/issues/1023#issuecomment-408128365" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP SYN Retransmits out of all retransmits", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Cluster", + "uid": "ff635a025bcfea7bc3dd4f508990a3e9", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml new file mode 100644 index 0000000000..8d420d7a4f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -0,0 +1,1196 @@ +{{- /* +Generated from 'controller-manager' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "controller-manager" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + controller-manager.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Controller Manager", + "uid": "72e0e05bef5099e5f049b05fdc429ed4", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml new file mode 100644 index 0000000000..0eeedc6299 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml @@ -0,0 +1,1229 @@ +{{- /* +Generated from 'etcd' from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "etcd" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + etcd.json: |- + { + "annotations": { + "list": [] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 42, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"$cluster\"}[$__rate_interval])))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer round trip time", + "metric": "etcd_network_peer_round_trip_time_seconds_bucket", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Peer round trip time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:925", + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:926", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + "etcd-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": {{ if (or .Values.grafana.sidecar.dashboards.multicluster.global.enabled .Values.grafana.sidecar.dashboards.multicluster.etcd.enabled) }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 2, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "etcd", + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", + "version": 215 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml new file mode 100644 index 0000000000..d2609140cf --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml @@ -0,0 +1,635 @@ +{{- /* +Generated from 'grafana-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "grafana-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + grafana-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + + ], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3085, + "iteration": 1631554945276, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Firing Alerts", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dashboards", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "displayMode": "auto" + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "showHeader": true + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_build_info{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Build Info", + "transformations": [ + { + "id": "labelsToFields", + "options": { + + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "branch": true, + "container": true, + "goversion": true, + "namespace": true, + "pod": true, + "revision": true + }, + "indexByName": { + "Time": 7, + "Value": 11, + "branch": 4, + "container": 8, + "edition": 2, + "goversion": 6, + "instance": 1, + "job": 0, + "namespace": 9, + "pod": 10, + "revision": 5, + "version": 3 + }, + "renameByName": { + + } + } + } + ], + "type": "table" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ", + "interval": "", + "legendFormat": "{{`{{`}}status_code{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "RPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:157", + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:210", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:211", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 30, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": [ + "default/grafana" + ], + "value": [ + "default/grafana" + ] + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, job)", + "refId": "Billing Admin-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "instance", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, instance)", + "refId": "Billing Admin-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Grafana Overview", + "uid": "6be0s85Mk", + "version": 2 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml new file mode 100644 index 0000000000..7ecca76f23 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml @@ -0,0 +1,1534 @@ +{{- /* +Generated from 'k8s-coredns' from ../files/dashboards/k8s-coredns.json +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.coreDns.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-coredns" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-coredns.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for the CoreDNS DNS server with updated metrics for version 1.7.0+. Based on the CoreDNS dashboard by buhay.", + "editable": true, + "gnetId": 12539, + "graphTooltip": 0, + "iteration": 1603798405693, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "CoreDNS.io", + "type": "link", + "url": "https://coredns.io" + } + ], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (total)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + }, + { + "alias": "other", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type) or \nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by qtype)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{zone}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by zone)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_do_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_do_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DO", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (DO bit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 2 + }, + { + "alias": "tcp:99 ", + "yaxis": 2 + }, + { + "alias": "tcp:50", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 1 + }, + { + "alias": "tcp:99 ", + "yaxis": 1 + }, + { + "alias": "tcp:50", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size,tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode) or\nsum(rate(coredns_dns_responses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{rcode}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (by rcode)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le, job))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (duration)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + }, + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{job=\"coredns\",instance=~\"$instance\"}) by (type) or\nsum(coredns_cache_entries{job=\"coredns\",instance=~\"$instance\"}) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (size)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "misses", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "hits:{{"{{type}}"}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_cache_misses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "misses", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (hitrate)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "dns", + "coredns" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(up{job=\"coredns\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=\"coredns\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "CoreDNS", + "uid": "vkQ0UHxik", + "version": 2 + } +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml new file mode 100644 index 0000000000..93ee57db93 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -0,0 +1,3088 @@ +{{- /* +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Namespace", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 20, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 21, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 22, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml new file mode 100644 index 0000000000..9c295831a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-multicluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-multicluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-multicluster.json: |- + {{`{"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster:node_cpu:ratio_rate5m","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Cluster","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Multi-Cluster","uid":"b59e6c9f2fcbe2e16d77fc492374cc4f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml new file mode 100644 index 0000000000..1c32c9c02e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -0,0 +1,2797 @@ +{{- /* +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml new file mode 100644 index 0000000000..e60a42d747 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -0,0 +1,1026 @@ +{{- /* +Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-node" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-node.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml new file mode 100644 index 0000000000..80fab51c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -0,0 +1,2469 @@ +{{- /* +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.25, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Throttling", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Throttling", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (WSS)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage (WSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Pod - Read & Writes)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Containers)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml new file mode 100644 index 0000000000..d77170afd8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-cluster' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-cluster.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - avg(rate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode=\"idle\"}[1m]))","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:windows_node_memory_MemFreeCached_bytes:sum{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Private Working Set)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Namespace","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Cluster(Windows)","uid":"4d08557fd9391b100730f2494bccac68","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml new file mode 100644 index 0000000000..13a1fc3abd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-namespace' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-namespace.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Namespace(Windows)","uid":"490b402361724ab1d4c45666c1fa9b6f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml new file mode 100644 index 0000000000..6686e54053 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-pod' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-pod.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":6,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum by (container) (rate(windows_container_network_received_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Received : {{ container }}","refId":"A"},{"expr":"sort_desc(sum by (container) (rate(windows_container_network_transmitted_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Transmitted : {{ container }}","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network I/O","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Pod","multi":false,"name":"pod","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\",namespace=\"$namespace\"}, pod)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Pod(Windows)","uid":"40597a704a610e936dc6ed374a7ce023","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml new file mode 100644 index 0000000000..e2a63ae208 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -0,0 +1,2024 @@ +{{- /* +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Pod", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}, workload)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml new file mode 100644 index 0000000000..95d758ea2d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -0,0 +1,2189 @@ +{{- /* +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Workload", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..d9ce9d738c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-cluster-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-cluster-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\"} * node:windows_node_num_cpu:sum{cluster=\"$cluster\"} / scalar(sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"}))","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:ratio{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O Pages)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(node:windows_node:sum{cluster=\"$cluster\"})","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (instance)(node:windows_node_filesystem_usage:{cluster=\"$cluster\"})\n","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Capacity","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Cluster(Windows)","uid":"53a43377ec9aaf2ff64dfc7a1f539334","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml new file mode 100644 index 0000000000..a7608496a3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-node-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-node-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (core) (irate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode!=\"idle\", instance=\"$instance\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{core}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage Per Core","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Memory","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation %","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"max(\n windows_os_visible_memory_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n - windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"memory used","refId":"A"},{"expr":"max(node:windows_node_memory_totalCached_bytes:sum{cluster=\"$cluster\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory cached","refId":"B"},{"expr":"max(windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory free","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Swap IO","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O) Pages","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"read","yaxis":1},{"alias":"io time","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"max(rate(windows_logical_disk_read_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"read","refId":"A"},{"expr":"max(rate(windows_logical_disk_write_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"written","refId":"B"},{"expr":"max(rate(windows_logical_disk_read_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]) + rate(windows_logical_disk_write_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ms","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Saturation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Net","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}\n","format":"time_series","legendFormat":"{{volume}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(windows_system_system_up_time{cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Node(Windows)","uid":"96e7484b0bb53b74fbc2bcb7723cd40b","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml new file mode 100644 index 0000000000..74a5303f8f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml @@ -0,0 +1,2256 @@ +{{- /* +Generated from 'kubelet' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubelet.enabled" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "kubelet" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + kubelet.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_node_name{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Running Kubelets", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Pods", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Containers", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 5, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Actual Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Desired Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 7, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Config Error Count", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 77 + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 77 + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 77 + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Kubelet", + "uid": "3138fa155d5915769fbded898ac09fd9", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml new file mode 100644 index 0000000000..f5c72844fb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -0,0 +1,1464 @@ +{{- /* +Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-pod.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "pod", + "value": "pod" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Pods)", + "uid": "8b7a8b326d7a6f1f04244066368c67af", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml new file mode 100644 index 0000000000..801b09c265 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -0,0 +1,1736 @@ +{{- /* +Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-workload.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "workload", + "value": "workload" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=$__cell", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Workload)", + "uid": "bbb2a765a623ae38130206c7d94a160f", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..9869a3d3e0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -0,0 +1,1063 @@ +{{- /* +Generated from 'node-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-cluster-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "((\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} instance {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"})))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Cluster", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml new file mode 100644 index 0000000000..75e69afa22 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -0,0 +1,1089 @@ +{{- /* +Generated from 'node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major page Faults", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_exporter_build_info{job=\"node-exporter\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Node", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml new file mode 100644 index 0000000000..fe11875324 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml @@ -0,0 +1,1073 @@ +{{- /* +Generated from 'nodes-darwin' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.darwin.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes-darwin" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes-darwin.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Physical Memory", + "refId": "A" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory Used", + "refId": "B" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "App Memory", + "refId": "C" + }, + { + "expr": "node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Wired Memory", + "refId": "D" + }, + { + "expr": "node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compressed", + "refId": "E" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "(\n (\n avg(node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"}) -\n avg(node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"})\n ) /\n avg(node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"})\n)\n*\n100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / MacOS", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml new file mode 100644 index 0000000000..0da40a7b99 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml @@ -0,0 +1,1066 @@ +{{- /* +Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.linux.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"})\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname!=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / Nodes", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml new file mode 100644 index 0000000000..4d1e33208b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -0,0 +1,587 @@ +{{- /* +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml new file mode 100644 index 0000000000..9a7e7d0603 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml @@ -0,0 +1,1228 @@ +{{- /* +Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "pod-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + pod-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 8, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Pod", + "uid": "7a18067ce943a40ae25454675c19ff5c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml new file mode 100644 index 0000000000..5c11900e69 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -0,0 +1,1674 @@ +{{- /* +Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus-remote-write" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus-remote-write.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} != 0)\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n, 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Min Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Desired Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shards", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shard Details", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}consumer{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Remote Write Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Segments", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Retried Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Enqueue Retries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Misc. Rates", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": true, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "url", + "options": [ + + ], + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Remote Write", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml new file mode 100644 index 0000000000..27f7c44e2c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml @@ -0,0 +1,1235 @@ +{{- /* +Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "s" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}scrape_job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}interval{{`}}`}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded body size limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}slice{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=\"prometheus-k8s\",namespace=\"monitoring\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Overview", + "uid": "", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml new file mode 100644 index 0000000000..410812451e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml @@ -0,0 +1,1276 @@ +{{- /* +Generated from 'proxy' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "proxy" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + proxy.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rules Sync Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rule Sync Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Proxy", + "uid": "632e265de029684c40b21cb76bca4f94", + "version": 0 + } +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml new file mode 100644 index 0000000000..ee0cf08b2f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml @@ -0,0 +1,1118 @@ +{{- /* +Generated from 'scheduler' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "scheduler" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + scheduler.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml new file mode 100644 index 0000000000..5aafccdebe --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml @@ -0,0 +1,1438 @@ +{{- /* +Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "workload-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + workload-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 14, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Workload", + "uid": "728bf77cc1166d2f3133bf25846876cc", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/namespaces.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/namespaces.yaml new file mode 100644 index 0000000000..39ed210ed4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/namespaces.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + name: {{ .Values.grafana.defaultDashboards.namespace }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + annotations: +{{- if not .Values.grafana.defaultDashboards.cleanupOnUninstall }} + helm.sh/resource-policy: "keep" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl new file mode 100644 index 0000000000..6ae9dc72e6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl @@ -0,0 +1,7 @@ +{{/* Generate basic labels for prometheus-operator */}} +{{- define "kube-prometheus-stack.prometheus-operator.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app: {{ template "kube-prometheus-stack.name" . }}-operator +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl new file mode 100644 index 0000000000..f419caf54b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl @@ -0,0 +1,6 @@ +{{/* Generate basic labels for prometheus-operator-webhook */}} +{{- define "kube-prometheus-stack.prometheus-operator-webhook.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator-webhook +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml new file mode 100644 index 0000000000..054eac4a77 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml @@ -0,0 +1,143 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.annotations | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.prometheusOperator.admissionWebhooks.deployment.replicas }} + revisionHistoryLimit: {{ .Values.prometheusOperator.admissionWebhooks.deployment.revisionHistoryLimit }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podLabels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: prometheus-operator-admission-webhook + {{- $operatorRegistry := .Values.global.imageRegistry | default .Values.prometheusOperator.admissionWebhooks.deployment.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + - --log-format={{ .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + - --log-level={{ .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - "--web.enable-tls=true" + - "--web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }}" + - "--web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }}" + - "--web.listen-address=:{{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }}" + - "--web.tls-min-version={{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.tlsMinVersion }}" + ports: + - containerPort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.failureThreshold }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.containerSecurityContext | indent 12 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + volumeMounts: + - name: tls-secret + mountPath: /cert + readOnly: true + volumes: + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }}-webhook + automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml new file mode 100644 index 0000000000..52dd78f624 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget -}} +apiVersion: policy/v1{{ ternary "" "beta1" ($.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml new file mode 100644 index 0000000000..b06c129123 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.admissionWebhooks.deployment.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml new file mode 100644 index 0000000000..55511da36b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ template "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | indent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml new file mode 100644 index 0000000000..f7543b0f1a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml new file mode 100644 index 0000000000..4e3b0d9225 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml new file mode 100644 index 0000000000..b81257c168 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") (or .Values.global.cattle.psp.enabled .Values.global.rbac.pspEnabled) }} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} + - apiGroups: ['policy'] +{{- else }} + - apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml new file mode 100644 index 0000000000..4cf1335b22 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml new file mode 100644 index 0000000000..baed83db48 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -0,0 +1,73 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: create + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | replace "\n" "," }} + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + {{- with .Values.prometheusOperator.admissionWebhooks.createSecretJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml new file mode 100644 index 0000000000..5639cc9e80 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -0,0 +1,74 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: patch + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - patch + - --webhook-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --patch-failure-policy={{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- with .Values.prometheusOperator.admissionWebhooks.patchWebhookJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml new file mode 100644 index 0000000000..864deb52a0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml new file mode 100644 index 0000000000..076c467004 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml new file mode 100644 index 0000000000..0113b6a5d8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,47 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- if .Values.global.rbac.pspAnnotations }} +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml new file mode 100644 index 0000000000..f15abf4395 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml new file mode 100644 index 0000000000..30bde920b6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml new file mode 100644 index 0000000000..02594547d1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..da01f3b57e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/mutate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..4827871cca --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/validate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/certmanager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/certmanager.yaml new file mode 100644 index 0000000000..cb27e49f48 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/certmanager.yaml @@ -0,0 +1,55 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled -}} +{{- if not .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef -}} +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }} + issuerRef: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.kube-prometheus-stack" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + ca: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert +{{- end }} +--- +# generate a server certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }} + issuerRef: + {{- if .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef }} + {{- toYaml .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef | nindent 4 }} + {{- else }} + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + {{- end }} + dnsNames: + {{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | splitList "\n" | toYaml | nindent 4 }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..07e2e99967 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml @@ -0,0 +1,40 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort | quote }} + {{- else }} + - port: "8080" + {{- end }} + protocol: "TCP" + {{- if not .Values.prometheusOperator.tls.enabled }} + rules: + http: + - method: "GET" + path: "/metrics" + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrole.yaml new file mode 100644 index 0000000000..fd11b69eed --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrole.yaml @@ -0,0 +1,109 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - alertmanagers/finalizers + - alertmanagers/status + - alertmanagerconfigs + - prometheuses + - prometheuses/finalizers + - prometheuses/status + - prometheusagents + - prometheusagents/finalizers + - prometheusagents/status + - thanosrulers + - thanosrulers/finalizers + - thanosrulers/status + - scrapeconfigs + - servicemonitors + - podmonitors + - probes + - prometheusrules + verbs: + - '*' +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - '*' +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete +- apiGroups: + - "" + resources: + - services + - services/finalizers + - endpoints + verbs: + - get + - create + - update + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - patch + - create +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get +{{- if .Capabilities.APIVersions.Has "discovery.k8s.io/v1/EndpointSlice" }} +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml new file mode 100644 index 0000000000..ad9e3ef6c5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/deployment.yaml new file mode 100644 index 0000000000..8a01b2912a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/deployment.yaml @@ -0,0 +1,204 @@ +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- $defaultKubeletSvcName := printf "%s-kubelet" (include "kube-prometheus-stack.fullname" .) }} +{{- if .Values.prometheusOperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.labels }} +{{ toYaml .Values.prometheusOperator.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.annotations | indent 4 }} +{{- end }} +spec: + replicas: 1 + revisionHistoryLimit: {{ .Values.prometheusOperator.revisionHistoryLimit }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- with .Values.prometheusOperator.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.podLabels }} +{{ toYaml .Values.prometheusOperator.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: {{ template "kube-prometheus-stack.name" . }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- $configReloaderRegistry := $base_registry | default .Values.prometheusOperator.prometheusConfigReloader.image.registry -}} + {{- $operatorRegistry := $base_registry | default .Values.prometheusOperator.image.registry -}} + {{- $thanosRegistry := $base_registry | default .Values.prometheusOperator.thanosImage.registry -}} + {{- if .Values.prometheusOperator.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.kubeletService.enabled }} + - --kubelet-service={{ .Values.prometheusOperator.kubeletService.namespace }}/{{ default $defaultKubeletSvcName .Values.prometheusOperator.kubeletService.name }} + {{- end }} + {{- if .Values.prometheusOperator.logFormat }} + - --log-format={{ .Values.prometheusOperator.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.logLevel }} + - --log-level={{ .Values.prometheusOperator.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - --deny-namespaces={{ tpl (.Values.prometheusOperator.denyNamespaces | join ",") $ }} + {{- end }} + {{- with $.Values.prometheusOperator.namespaces }} + {{- $namespaces := list }} + {{- if .releaseNamespace }} + {{- $namespaces = append $namespaces $namespace }} + {{- end }} + {{- if .additional }} + {{- range $ns := .additional }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + - --localhost=127.0.0.1 + {{- if .Values.prometheusOperator.prometheusDefaultBaseImage }} + - --prometheus-default-base-image={{ $base_registry | default .Values.prometheusOperator.prometheusDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.prometheusDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + - --alertmanager-default-base-image={{ $base_registry | default .Values.prometheusOperator.alertmanagerDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + {{- else }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }} + {{- end }} + - --config-reloader-cpu-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).cpu) | default 0 }} + - --config-reloader-cpu-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).cpu) | default 0 }} + - --config-reloader-memory-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).memory) | default 0 }} + - --config-reloader-memory-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).memory) | default 0 }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.enableProbe }} + - --enable-config-reloader-probes=true + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceNamespaces }} + - --alertmanager-instance-namespaces={{ .Values.prometheusOperator.alertmanagerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceSelector }} + - --alertmanager-instance-selector={{ .Values.prometheusOperator.alertmanagerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerConfigNamespaces }} + - --alertmanager-config-namespaces={{ .Values.prometheusOperator.alertmanagerConfigNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceNamespaces }} + - --prometheus-instance-namespaces={{ .Values.prometheusOperator.prometheusInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceSelector }} + - --prometheus-instance-selector={{ .Values.prometheusOperator.prometheusInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.thanosImage.sha }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}@sha256:{{ .Values.prometheusOperator.thanosImage.sha }} + {{- else }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceNamespaces }} + - --thanos-ruler-instance-namespaces={{ .Values.prometheusOperator.thanosRulerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceSelector }} + - --thanos-ruler-instance-selector={{ .Values.prometheusOperator.thanosRulerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.secretFieldSelector }} + - --secret-field-selector={{ tpl (.Values.prometheusOperator.secretFieldSelector) $ }} + {{- end }} + {{- if .Values.prometheusOperator.clusterDomain }} + - --cluster-domain={{ .Values.prometheusOperator.clusterDomain }} + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - --web.enable-tls=true + - --web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }} + - --web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }} + - --web.listen-address=:{{ .Values.prometheusOperator.tls.internalPort }} + - --web.tls-min-version={{ .Values.prometheusOperator.tls.tlsMinVersion }} + ports: + - containerPort: {{ .Values.prometheusOperator.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + env: + {{- range $key, $value := .Values.prometheusOperator.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.containerSecurityContext | indent 12 }} + volumeMounts: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + mountPath: /cert + readOnly: true + {{- end }} + {{- with .Values.prometheusOperator.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + {{- end }} + {{- with .Values.prometheusOperator.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.prometheusOperator.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.tolerations }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/networkpolicy.yaml new file mode 100644 index 0000000000..cfd5b0b8c7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/networkpolicy.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + egress: + - {} + ingress: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort }} + {{- else }} + - port: 8080 + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml new file mode 100644 index 0000000000..61bc3d9040 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.operator.fullname" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..40e0fc5c15 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp.yaml new file mode 100644 index 0000000000..28a9075d3e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: {{ .Values.prometheusOperator.hostNetwork }} + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/service.yaml new file mode 100644 index 0000000000..d45ab22d08 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/service.yaml @@ -0,0 +1,57 @@ +{{- if .Values.prometheusOperator.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.service.labels }} +{{ toYaml .Values.prometheusOperator.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/serviceaccount.yaml new file mode 100644 index 0000000000..4f84974f9b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +automountServiceAccountToken: {{ .Values.prometheusOperator.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/servicemonitor.yaml new file mode 100644 index 0000000000..cbe79e1253 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- with .Values.prometheusOperator.serviceMonitor.additionalLabels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheusOperator.serviceMonitor | nindent 2 }} + endpoints: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: https + scheme: https + tlsConfig: + serverName: {{ template "kube-prometheus-stack.operator.fullname" . }} + ca: + secret: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + key: {{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}ca.crt{{ else }}ca{{ end }} + optional: false + {{- else }} + - port: http + {{- end }} + honorLabels: true + {{- if .Values.prometheusOperator.serviceMonitor.interval }} + interval: {{ .Values.prometheusOperator.serviceMonitor.interval }} + {{- end }} + metricRelabelings: + {{- if .Values.prometheusOperator.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.prometheusOperator.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheusOperator.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheusOperator.serviceMonitor.relabelings | indent 6 }} +{{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..f225d16dde --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.prometheusOperator.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + {{- with .Values.prometheusOperator.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-prometheus-stack.name" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/_rules.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/_rules.tpl new file mode 100644 index 0000000000..4a8213d089 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/_rules.tpl @@ -0,0 +1,44 @@ +{{- /* +Generated file. Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- define "rules.names" }} +rules: + - "alertmanager.rules" + - "config-reloaders" + - "etcd" + - "general.rules" + - "k8s.rules.container-cpu-usage-seconds-total" + - "k8s.rules.container-memory-cache" + - "k8s.rules.container-memory-rss" + - "k8s.rules.container-memory-swap" + - "k8s.rules.container-memory-working-set-bytes" + - "k8s.rules.container-resource" + - "k8s.rules.pod-owner" + - "kube-apiserver-availability.rules" + - "kube-apiserver-burnrate.rules" + - "kube-apiserver-histogram.rules" + - "kube-apiserver-slos" + - "kube-prometheus-general.rules" + - "kube-prometheus-node-recording.rules" + - "kube-scheduler.rules" + - "kube-state-metrics" + - "kubelet.rules" + - "kubernetes-apps" + - "kubernetes-resources" + - "kubernetes-storage" + - "kubernetes-system" + - "kubernetes-system-kube-proxy" + - "kubernetes-system-apiserver" + - "kubernetes-system-kubelet" + - "kubernetes-system-controller-manager" + - "kubernetes-system-scheduler" + - "node-exporter.rules" + - "node-exporter" + - "node.rules" + - "node-network" + - "prometheus-operator" + - "prometheus" + - "windows.node.rules" + - "windows.pod.rules" +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml new file mode 100644 index 0000000000..bff930981a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-relabel-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alert-relabel-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml new file mode 100644 index 0000000000..2fe8fdb816 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alertmanager-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs) . | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml new file mode 100644 index 0000000000..cb4aabaa7b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml @@ -0,0 +1,43 @@ +{{- if or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-additional-prometheus-rules + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- if .Values.additionalPrometheusRulesMap }} +{{- range $prometheusRuleName, $prometheusRule := .Values.additionalPrometheusRulesMap }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ $prometheusRuleName }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $prometheusRule.additionalLabels }} +{{ toYaml $prometheusRule.additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml $prometheusRule.groups| indent 8 }} +{{- end }} +{{- else }} +{{- range .Values.additionalPrometheusRules }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml .groups| indent 8 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml new file mode 100644 index 0000000000..ebdf766fde --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-scrape-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if eq ( typeOf .Values.prometheus.prometheusSpec.additionalScrapeConfigs ) "string" }} + additional-scrape-configs.yaml: {{ tpl .Values.prometheus.prometheusSpec.additionalScrapeConfigs $ | b64enc | quote }} +{{- else }} + additional-scrape-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalScrapeConfigs) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..74d61d7c13 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} +spec: + endpointSelector: + {{- if .Values.prometheus.networkPolicy.cilium.endpointSelector }} + {{- toYaml .Values.prometheus.networkPolicy.cilium.endpointSelector | nindent 4 }} + {{- else }} + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.egress }} + egress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.egress | nindent 4 }} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.ingress }} + ingress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.ingress | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrole.yaml new file mode 100644 index 0000000000..3585b5db11 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrole.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +# This permission are not in the kube-prometheus repo +# they're grabbed from https://github.com/prometheus/prometheus/blob/master/documentation/examples/rbac-setup.yml +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - "networking.k8s.io" + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +{{- if .Values.prometheus.additionalRulesForClusterRole }} +{{ toYaml .Values.prometheus.additionalRulesForClusterRole | indent 0 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrolebinding.yaml new file mode 100644 index 0000000000..9fc4f65da4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/csi-secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/csi-secret.yaml new file mode 100644 index 0000000000..e05382f633 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/csi-secret.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.secretProviderClass }} +--- +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +spec: +{{ toYaml .Values.prometheus.prometheusSpec.thanos.secretProviderClass | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/extrasecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/extrasecret.yaml new file mode 100644 index 0000000000..17f3478a46 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.prometheus.extraSecret.data -}} +{{- $secretName := printf "prometheus-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.prometheus.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.extraSecret.annotations }} + annotations: +{{ toYaml .Values.prometheus.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.prometheus.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingress.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingress.yaml new file mode 100644 index 0000000000..d2f6af5dd1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}} + {{- $pathType := .Values.prometheus.ingress.pathType | default "ImplementationSpecific" -}} + {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" -}} + {{- $servicePort := .Values.prometheus.ingress.servicePort | default .Values.prometheus.service.port -}} + {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} + {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} + {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} + {{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.ingress.annotations) . | nindent 4 }} +{{- end }} + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.ingress.labels }} +{{ toYaml .Values.prometheus.ingress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.ingress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.ingress.hosts }} + {{- range $host := .Values.prometheus.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.ingress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml new file mode 100644 index 0000000000..3f507cfa9f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosIngress.enabled }} +{{- $pathType := .Values.prometheus.thanosIngress.pathType | default "" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "thanos-discovery" }} +{{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} +{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} +{{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.thanosIngress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.thanosIngress.annotations) . | nindent 4 }} +{{- end }} + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-gateway + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosIngress.labels }} +{{ toYaml .Values.prometheus.thanosIngress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.thanosIngress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.thanosIngress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.thanosIngress.hosts }} + {{- range $host := .Values.prometheus.thanosIngress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.thanosIngress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.thanosIngress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressperreplica.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressperreplica.yaml new file mode 100644 index 0000000000..1d76d135c8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled .Values.prometheus.ingressPerReplica.enabled }} +{{- $pathType := .Values.prometheus.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $servicePort := .Values.prometheus.servicePerReplica.port -}} +{{- $ingressValues := .Values.prometheus.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" $ }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/networkpolicy.yaml new file mode 100644 index 0000000000..1296a79063 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/networkpolicy.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + {{- if .Values.prometheus.networkPolicy.egress }} + egress: + {{- toYaml .Values.prometheus.networkPolicy.egress | nindent 4 }} + {{- end }} + {{- if .Values.prometheus.networkPolicy.ingress }} + ingress: + {{- toYaml .Values.prometheus.networkPolicy.ingress | nindent 4 }} + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + {{- if .Values.prometheus.networkPolicy.podSelector }} + {{- toYaml .Values.prometheus.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/nginx-config.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/nginx-config.yaml new file mode 100644 index 0000000000..e4d91f9a9e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/nginx-config.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-nginx-proxy-config + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + server { + listen 8081; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:9090/; + + sub_filter_once off; + sub_filter 'var PATH_PREFIX = "";' 'var PATH_PREFIX = ".";'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podDisruptionBudget.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podDisruptionBudget.yaml new file mode 100644 index 0000000000..48f3f1f5a6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podDisruptionBudget.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.prometheus.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.prometheus.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.prometheus.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.prometheus.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podmonitors.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podmonitors.yaml new file mode 100644 index 0000000000..4e748c23b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podmonitors.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalPodMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalPodMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + podMetricsEndpoints: +{{ toYaml .podMetricsEndpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .sampleLimit }} + sampleLimit: {{ .sampleLimit }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/prometheus.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/prometheus.yaml new file mode 100644 index 0000000000..5c3c8d4d1f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/prometheus.yaml @@ -0,0 +1,472 @@ +{{- if .Values.prometheus.enabled }} +{{- if .Values.prometheus.agentMode }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: PrometheusAgent +{{- else }} +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +{{- end }} +metadata: + name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +spec: +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.alertingEndpoints .Values.alertmanager.enabled) }} + alerting: + alertmanagers: +{{- if .Values.prometheus.prometheusSpec.alertingEndpoints }} +{{ toYaml .Values.prometheus.prometheusSpec.alertingEndpoints | indent 6 }} +{{- else if .Values.alertmanager.enabled }} + - namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + pathPrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.scheme }} + scheme: {{ .Values.alertmanager.alertmanagerSpec.scheme }} + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.tlsConfig }} + tlsConfig: +{{ toYaml .Values.alertmanager.alertmanagerSpec.tlsConfig | indent 10 }} + {{- end }} + apiVersion: {{ .Values.alertmanager.apiVersion }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.apiserverConfig }} + apiserverConfig: +{{ toYaml .Values.prometheus.prometheusSpec.apiserverConfig | indent 4}} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.prometheus.prometheusSpec.image.registry -}} + {{- if and .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}" + {{- end }} + version: {{ default .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.version }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.prometheus.prometheusSpec.additionalArgs | indent 4}} +{{- end -}} +{{- if .Values.prometheus.prometheusSpec.externalLabels }} + externalLabels: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusExternalLabelNameClear }} + prometheusExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.prometheusExternalLabelName }} + prometheusExternalLabelName: "{{ .Values.prometheus.prometheusSpec.prometheusExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.replicaExternalLabelNameClear }} + replicaExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.replicaExternalLabelName }} + replicaExternalLabelName: "{{ .Values.prometheus.prometheusSpec.replicaExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} + enableRemoteWriteReceiver: {{ .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalUrl }} + externalUrl: "{{ tpl .Values.prometheus.prometheusSpec.externalUrl . }}" +{{- else if and .Values.prometheus.ingress.enabled .Values.prometheus.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.prometheus.ingress.hosts 0) . }}{{ .Values.prometheus.prometheusSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "kube-prometheus-stack.namespace" . }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.nodeSelector }} +{{ toYaml .Values.prometheus.prometheusSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.prometheus.prometheusSpec.paused }} + replicas: {{ .Values.prometheus.prometheusSpec.replicas }} + shards: {{ .Values.prometheus.prometheusSpec.shards }} + logLevel: {{ .Values.prometheus.prometheusSpec.logLevel }} + logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} + listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} +{{- if not .Values.prometheus.agentMode }} + enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.web }} + web: +{{ toYaml .Values.prometheus.prometheusSpec.web | indent 4 }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.exemplars }} + exemplars: + {{ toYaml .Values.prometheus.prometheusSpec.exemplars | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableFeatures }} + enableFeatures: +{{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }} + - {{ tpl $enableFeatures $ }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeInterval }} + scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.prometheusSpec.scrapeTimeout }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.evaluationInterval }} + evaluationInterval: {{ .Values.prometheus.prometheusSpec.evaluationInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.resources }} + resources: +{{ toYaml .Values.prometheus.prometheusSpec.resources | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} + retention: {{ .Values.prometheus.prometheusSpec.retention | quote }} +{{- if .Values.prometheus.prometheusSpec.retentionSize }} + retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tsdb }} + tsdb: + {{- if .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + outOfOrderTimeWindow: {{ .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + {{- end }} +{{- end }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.walCompression false }} + walCompression: false +{{ else }} + walCompression: true +{{- end }} +{{- if .Values.prometheus.prometheusSpec.routePrefix }} + routePrefix: {{ .Values.prometheus.prometheusSpec.routePrefix | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.secrets }} + secrets: +{{ toYaml .Values.prometheus.prometheusSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.configMaps }} + configMaps: +{{ toYaml .Values.prometheus.prometheusSpec.configMaps | indent 4 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorSelector }} + serviceMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues }} + serviceMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + serviceMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector }} + serviceMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector | indent 4) . }} +{{ else }} + serviceMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorSelector }} + podMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues }} + podMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + podMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector }} + podMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector | indent 4) . }} +{{ else }} + podMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeSelector }} + probeSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.probeSelectorNilUsesHelmValues }} + probeSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + probeSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeNamespaceSelector }} + probeNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeNamespaceSelector | indent 4) . }} +{{ else }} + probeNamespaceSelector: {} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.remoteRead .Values.prometheus.prometheusSpec.additionalRemoteRead) }} + remoteRead: +{{- if .Values.prometheus.prometheusSpec.remoteRead }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteRead | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteRead }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteRead | indent 4 }} +{{- end }} +{{- end }} +{{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite) }} + remoteWrite: +{{- if .Values.prometheus.prometheusSpec.remoteWrite }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteWrite }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteWrite | indent 4 }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.prometheusSpec.securityContext | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleSelector | indent 4) . }} +{{- else if .Values.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigSelector }} + scrapeConfigSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.scrapeConfigSelectorNilUsesHelmValues }} + scrapeConfigSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + scrapeConfigSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector }} + scrapeConfigNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector | indent 4) . }} +{{ else }} + scrapeConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.storageSpec }} + storage: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.storageSpec | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMetadata }} + podMetadata: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMetadata | indent 4) . }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.query }} + query: +{{ toYaml .Values.prometheus.prometheusSpec.query | indent 4}} +{{- end }} +{{- if or .Values.prometheus.prometheusSpec.podAntiAffinity .Values.prometheus.prometheusSpec.affinity }} + affinity: +{{- if .Values.prometheus.prometheusSpec.affinity }} +{{ toYaml .Values.prometheus.prometheusSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.tolerations }} +{{ toYaml .Values.prometheus.prometheusSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.prometheus.prometheusSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} + additionalScrapeConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + key: additional-scrape-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.enabled }} + additionalScrapeConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.key }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if or .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + additionalAlertManagerConfigs: +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + key: additional-alertmanager-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + name: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.key }} + {{- if hasKey .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret "optional" }} + optional: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.optional }} + {{- end }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} + additionalAlertRelabelConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + key: additional-alert-relabel-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret }} + additionalAlertRelabelConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.key }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.containers }} + containers: +{{ tpl .Values.prometheus.prometheusSpec.containers $ | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.initContainers }} + initContainers: +{{ toYaml .Values.prometheus.prometheusSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.priorityClassName }} + priorityClassName: {{ .Values.prometheus.prometheusSpec.priorityClassName }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.thanos }} + thanos: +{{- with (omit .Values.prometheus.prometheusSpec.thanos "objectStorageConfig")}} +{{ toYaml . | indent 4 }} +{{- end }} +{{- if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).existingSecret) }} + objectStorageConfig: + key: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.name }}" +{{- else if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).secret) }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.disableCompaction }} + disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }} +{{- end }} +{{- end }} + portName: {{ .Values.prometheus.prometheusSpec.portName }} +{{- if .Values.prometheus.prometheusSpec.volumes }} + volumes: +{{ toYaml .Values.prometheus.prometheusSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.prometheus.prometheusSpec.volumeMounts | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs }} + arbitraryFSAccessThroughSMs: +{{ toYaml .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorLabels }} + overrideHonorLabels: {{ .Values.prometheus.prometheusSpec.overrideHonorLabels }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} + overrideHonorTimestamps: {{ .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + ignoreNamespaceSelectors: {{ .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} + enforcedNamespaceLabel: {{ .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} +{{- $prometheusDefaultRulesExcludedFromEnforce := (include "rules.names" .) | fromYaml }} +{{- if not .Values.prometheus.agentMode }} + prometheusRulesExcludedFromEnforce: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - ruleNamespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + ruleName: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce }} +{{ toYaml .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce | indent 4 }} +{{- end }} +{{- end }} + excludedFromEnforcement: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - group: monitoring.coreos.com + resource: prometheusrules + namespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + name: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.excludedFromEnforcement }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.excludedFromEnforcement | indent 4) . }} +{{- end }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.queryLogFile }} + queryLogFile: {{ .Values.prometheus.prometheusSpec.queryLogFile }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.sampleLimit }} + sampleLimit: {{ .Values.prometheus.prometheusSpec.sampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} + enforcedKeepDroppedTargets: {{ .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedSampleLimit }} + enforcedSampleLimit: {{ .Values.prometheus.prometheusSpec.enforcedSampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedTargetLimit }} + enforcedTargetLimit: {{ .Values.prometheus.prometheusSpec.enforcedTargetLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelLimit }} + enforcedLabelLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} + enforcedLabelNameLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit}} + enforcedLabelValueLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} + allowOverlappingBlocks: {{ .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.minReadySeconds }} + minReadySeconds: {{ .Values.prometheus.prometheusSpec.minReadySeconds }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} + maximumStartupDurationSeconds: {{ .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} +{{- end }} + hostNetwork: {{ .Values.prometheus.prometheusSpec.hostNetwork }} +{{- if .Values.prometheus.prometheusSpec.hostAliases }} + hostAliases: +{{ toYaml .Values.prometheus.prometheusSpec.hostAliases | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tracingConfig }} + tracingConfig: +{{ toYaml .Values.prometheus.prometheusSpec.tracingConfig | indent 4 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrole.yaml new file mode 100644 index 0000000000..71476cd18b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrole.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..a393928c78 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp.yaml new file mode 100644 index 0000000000..62d3854151 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' +{{- if .Values.prometheus.podSecurityPolicy.volumes }} +{{ toYaml .Values.prometheus.podSecurityPolicy.volumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- if .Values.prometheus.podSecurityPolicy.allowedCapabilities }} + allowedCapabilities: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedCapabilities | indent 4 }} +{{- end }} +{{- if .Values.prometheus.podSecurityPolicy.allowedHostPaths }} + allowedHostPaths: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedHostPaths | indent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml new file mode 100644 index 0000000000..b66f052ade --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -0,0 +1,305 @@ +{{- /* +Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/alertmanager-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} +{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: alertmanager.rules + rules: +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedReload | default false) }} + - alert: AlertmanagerFailedReload + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Configuration has failed to load for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedreload + summary: Reloading an Alertmanager configuration has failed. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "AlertmanagerFailedReload" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedReload" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerMembersInconsistent | default false) }} + - alert: AlertmanagerMembersInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} has only found {{`{{`}} $value {{`}}`}} members of the {{`{{`}}$labels.job{{`}}`}} cluster. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagermembersinconsistent + summary: A member of an Alertmanager cluster has not found all other cluster members. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + < on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) group_left + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) (max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m])) + for: {{ dig "AlertmanagerMembersInconsistent" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerMembersInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedToSendAlerts | default false) }} + - alert: AlertmanagerFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} failed to send {{`{{`}} $value | humanizePercentage {{`}}`}} of notifications to {{`{{`}} $labels.integration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedtosendalerts + summary: An Alertmanager instance failed to send notifications. + expr: |- + ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerConfigInconsistent | default false) }} + - alert: AlertmanagerConfigInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have different configurations. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerconfiginconsistent + summary: Alertmanager instances within the same cluster have different configurations. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + count_values by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + ) + != 1 + for: {{ dig "AlertmanagerConfigInconsistent" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerConfigInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterDown | default false) }} + - alert: AlertmanagerClusterDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have been up for less than half of the last 5m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterdown + summary: Half or more of the Alertmanager instances within the same cluster are down. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + avg_over_time(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) < 0.5 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterCrashlooping | default false) }} + - alert: AlertmanagerClusterCrashlooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have restarted at least 5 times in the last 10m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclustercrashlooping + summary: Half or more of the Alertmanager instances within the same cluster are crashlooping. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + changes(process_start_time_seconds{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[10m]) > 4 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterCrashlooping" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterCrashlooping" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml new file mode 100644 index 0000000000..8416d6df40 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml @@ -0,0 +1,57 @@ +{{- /* +Generated from 'config-reloaders' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.configReloaders }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "config-reloaders" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: config-reloaders + rules: +{{- if not (.Values.defaultRules.disabled.ConfigReloaderSidecarErrors | default false) }} + - alert: ConfigReloaderSidecarErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders | indent 8 }} +{{- end }} + description: 'Errors encountered while the {{`{{`}}$labels.pod{{`}}`}} config-reloader sidecar attempts to sync config in {{`{{`}}$labels.namespace{{`}}`}} namespace. + + As a result, configuration for service running in {{`{{`}}$labels.pod{{`}}`}} may be stale and cannot be updated anymore.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/configreloadersidecarerrors + summary: config-reloader sidecar has not had a successful reload for 10m + expr: max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 + for: {{ dig "ConfigReloaderSidecarErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "ConfigReloaderSidecarErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml new file mode 100644 index 0000000000..a1d7a508f8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml @@ -0,0 +1,461 @@ +{{- /* +Generated from 'etcd' group from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.etcd }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: etcd + rules: +{{- if not (.Values.defaultRules.disabled.etcdMembersDown | default false) }} + - alert: etcdMembersDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster members are down. + expr: |- + max without (endpoint) ( + sum without (instance) (up{job=~".*etcd.*"} == bool 0) + or + count without (To) ( + sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + ) + ) + > 0 + for: {{ dig "etcdMembersDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMembersDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdInsufficientMembers | default false) }} + - alert: etcdInsufficientMembers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster has insufficient number of members. + expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2) + for: {{ dig "etcdInsufficientMembers" "for" "3m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdInsufficientMembers" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdNoLeader | default false) }} + - alert: etcdNoLeader + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.' + summary: etcd cluster has no leader. + expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 + for: {{ dig "etcdNoLeader" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdNoLeader" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfLeaderChanges | default false) }} + - alert: etcdHighNumberOfLeaderChanges + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' + summary: etcd cluster has high number of leader changes. + expr: increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4 + for: {{ dig "etcdHighNumberOfLeaderChanges" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfLeaderChanges" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 1 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 5 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdGRPCRequestsSlow | default false) }} + - alert: etcdGRPCRequestsSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.' + summary: etcd grpc requests are slow + expr: |- + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) + > 0.15 + for: {{ dig "etcdGRPCRequestsSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdGRPCRequestsSlow" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdMemberCommunicationSlow | default false) }} + - alert: etcdMemberCommunicationSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster member communication is slow. + expr: |- + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.15 + for: {{ dig "etcdMemberCommunicationSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMemberCommunicationSlow" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedProposals | default false) }} + - alert: etcdHighNumberOfFailedProposals + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of proposal failures. + expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + for: {{ dig "etcdHighNumberOfFailedProposals" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedProposals" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.5 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 1 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighCommitDurations | default false) }} + - alert: etcdHighCommitDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile commit durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.25 + for: {{ dig "etcdHighCommitDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighCommitDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseQuotaLowSpace | default false) }} + - alert: etcdDatabaseQuotaLowSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' + summary: etcd cluster database is running full. + expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95 + for: {{ dig "etcdDatabaseQuotaLowSpace" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseQuotaLowSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdExcessiveDatabaseGrowth | default false) }} + - alert: etcdExcessiveDatabaseGrowth + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.' + summary: etcd cluster database growing very fast. + expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"} + for: {{ dig "etcdExcessiveDatabaseGrowth" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdExcessiveDatabaseGrowth" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseHighFragmentationRatio | default false) }} + - alert: etcdDatabaseHighFragmentationRatio + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size in use on instance {{`{{`}} $labels.instance {{`}}`}} is {{`{{`}} $value | humanizePercentage {{`}}`}} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.' + runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation + summary: etcd database size in use is less than 50% of the actual allocated storage. + expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600 + for: {{ dig "etcdDatabaseHighFragmentationRatio" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseHighFragmentationRatio" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml new file mode 100644 index 0000000000..8aca0b85f5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml @@ -0,0 +1,125 @@ +{{- /* +Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: general.rules + rules: +{{- if not (.Values.defaultRules.disabled.TargetDown | default false) }} + - alert: TargetDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/targetdown + summary: One or more targets are unreachable. + expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10 + for: {{ dig "TargetDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "TargetDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.Watchdog | default false) }} + - alert: Watchdog + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + + This alert is always firing, therefore it should always be firing in Alertmanager + + and always fire against a receiver. There are integrations with various notification + + mechanisms that send a notification when this alert is not firing. For example the + + "DeadMansSnitch" integration in PagerDuty. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager is working properly. + expr: vector(1) + labels: + severity: {{ dig "Watchdog" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.InfoInhibitor | default false) }} + - alert: InfoInhibitor + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert that is used to inhibit info alerts. + + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + + other alerts. + + This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a + + severity of ''warning'' or ''critical'' starts firing on the same namespace. + + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: {{ dig "InfoInhibitor" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml new file mode 100644 index 0000000000..9de5f5bc9c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml @@ -0,0 +1,43 @@ +{{- /* +Generated from 'k8s.rules.container-cpu-usage-seconds-total' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerCpuUsageSecondsTotal }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-cpu-usage-seconds-total" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_cpu_usage_seconds_total + rules: + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml new file mode 100644 index 0000000000..323f41f9cb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-cache' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryCache }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-cache" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_cache + rules: + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml new file mode 100644 index 0000000000..312d73c889 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-rss' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryRss }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-rss" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_rss + rules: + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml new file mode 100644 index 0000000000..136595e801 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-swap' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemorySwap }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-swap" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_swap + rules: + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml new file mode 100644 index 0000000000..d308b7473a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-working-set-bytes' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryWorkingSetBytes }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-working-set-bytes" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_working_set_bytes + rules: + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml new file mode 100644 index 0000000000..2d896e59e4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml @@ -0,0 +1,168 @@ +{{- /* +Generated from 'k8s.rules.container-resource' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerResource }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-resource" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_resource + rules: + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml new file mode 100644 index 0000000000..4915b25e73 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml @@ -0,0 +1,107 @@ +{{- /* +Generated from 'k8s.rules.pod-owner' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sPodOwner }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.pod-owner" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.pod_owner + rules: + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) group_left(owner_name) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="{{ $kubeStateMetricsJob }}"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml new file mode 100644 index 0000000000..c61bd222ab --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -0,0 +1,237 @@ +{{- /* +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8s }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules + rules: + - expr: |- + sum by (cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( + 1, max by (replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml new file mode 100644 index 0000000000..6194e9c614 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -0,0 +1,273 @@ +{{- /* +Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverAvailability }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-availability.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - interval: 3m + name: kube-apiserver-availability.rules + rules: + - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30 + record: code_verb:apiserver_request_total:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (increase(apiserver_request_sli_duration_seconds_count{job="apiserver"}[1h])) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h])) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + # write too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + ( + # read too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + ) + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d) + labels: + verb: all + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + # too slow + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + ( + # too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml new file mode 100644 index 0000000000..e6666a6f41 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml @@ -0,0 +1,440 @@ +{{- /* +Generated from 'kube-apiserver-burnrate.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverBurnrate }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-burnrate.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-burnrate.rules + rules: + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml new file mode 100644 index 0000000000..d145341952 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml @@ -0,0 +1,53 @@ +{{- /* +Generated from 'kube-apiserver-histogram.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverHistogram }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-histogram.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-histogram.rules + rules: + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml new file mode 100644 index 0000000000..30ef9a4293 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -0,0 +1,159 @@ +{{- /* +Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverSlos }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-slos" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-slos + rules: +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) + and + sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) + and + sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 6h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 30m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) + and + sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 2h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) + and + sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "3h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 3d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 6h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml new file mode 100644 index 0000000000..fcf35f389b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml @@ -0,0 +1,49 @@ +{{- /* +Generated from 'kube-prometheus-general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusGeneral }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-general.rules + rules: + - expr: count without(instance, pod, node) (up == 1) + record: count:up1 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: count without(instance, pod, node) (up == 0) + record: count:up0 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml new file mode 100644 index 0000000000..7a0d202324 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml @@ -0,0 +1,93 @@ +{{- /* +Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-recording.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-node-recording.rules + rules: + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance) + record: instance:node_cpu:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) + record: instance:node_network_receive_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) + record: instance:node_network_transmit_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) + record: instance:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) + record: cluster:node_cpu:sum_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) + record: cluster:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml new file mode 100644 index 0000000000..c9d61ce37b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml @@ -0,0 +1,135 @@ +{{- /* +Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-scheduler.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-scheduler.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml new file mode 100644 index 0000000000..d1ad3cae5e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml @@ -0,0 +1,152 @@ +{{- /* +Generated from 'kube-state-metrics' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubeStateMetrics-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeStateMetrics }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-state-metrics" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-state-metrics + rules: +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsListErrors | default false) }} + - alert: KubeStateMetricsListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricslisterrors + summary: kube-state-metrics is experiencing errors in list operations. + expr: |- + (sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsListErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsWatchErrors | default false) }} + - alert: KubeStateMetricsWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricswatcherrors + summary: kube-state-metrics is experiencing errors in watch operations. + expr: |- + (sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsWatchErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardingMismatch | default false) }} + - alert: KubeStateMetricsShardingMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardingmismatch + summary: kube-state-metrics sharding is misconfigured. + expr: stdvar (kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) != 0 + for: {{ dig "KubeStateMetricsShardingMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardingMismatch" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardsMissing | default false) }} + - alert: KubeStateMetricsShardsMissing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardsmissing + summary: kube-state-metrics shards are missing. + expr: |- + 2^max(kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - 1 + - + sum( 2 ^ max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="{{ $kubeStateMetricsJob }}"}) ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + != 0 + for: {{ dig "KubeStateMetricsShardsMissing" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardsMissing" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml new file mode 100644 index 0000000000..39fdddf3fe --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml @@ -0,0 +1,65 @@ +{{- /* +Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubelet }} +{{- if (include "exporter.kubelet.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubelet.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubelet.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml new file mode 100644 index 0000000000..2a861a522c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -0,0 +1,568 @@ +{{- /* +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesApps }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-apps + rules: +{{- if not (.Values.defaultRules.disabled.KubePodCrashLooping | default false) }} + - alert: KubePodCrashLooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: 'Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is in waiting state (reason: "CrashLoopBackOff").' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodcrashlooping + summary: Pod is crash looping. + expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) >= 1 + for: {{ dig "KubePodCrashLooping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodCrashLooping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePodNotReady | default false) }} + - alert: KubePodNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodnotready + summary: Pod has been in a non-ready state for more than 15 minutes. + expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown|Failed"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left(owner_kind) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + for: {{ dig "KubePodNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentGenerationMismatch | default false) }} + - alert: KubeDeploymentGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentgenerationmismatch + summary: Deployment generation mismatch due to possible roll-back + expr: |- + kube_deployment_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeDeploymentGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentReplicasMismatch | default false) }} + - alert: KubeDeploymentReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_deployment_spec_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_deployment_status_replicas_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeDeploymentReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentRolloutStuck | default false) }} + - alert: KubeDeploymentRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Rollout of deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} is not progressing for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentrolloutstuck + summary: Deployment rollout is not progressing. + expr: |- + kube_deployment_status_condition{condition="Progressing", status="false",job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != 0 + for: {{ dig "KubeDeploymentRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetReplicasMismatch | default false) }} + - alert: KubeStatefulSetReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetreplicasmismatch + summary: StatefulSet has not matched the expected number of replicas. + expr: |- + ( + kube_statefulset_status_replicas_ready{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetGenerationMismatch | default false) }} + - alert: KubeStatefulSetGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetgenerationmismatch + summary: StatefulSet generation mismatch due to possible roll-back + expr: |- + kube_statefulset_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeStatefulSetGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetUpdateNotRolledOut | default false) }} + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetupdatenotrolledout + summary: StatefulSet update has not been rolled out. + expr: |- + ( + max without (revision) ( + kube_statefulset_status_current_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetUpdateNotRolledOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetUpdateNotRolledOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetRolloutStuck | default false) }} + - alert: KubeDaemonSetRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} has not finished or progressed for at least 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetrolloutstuck + summary: DaemonSet rollout is stuck. + expr: |- + ( + ( + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + 0 + ) or ( + kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeDaemonSetRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeContainerWaiting | default false) }} + - alert: KubeContainerWaiting + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontainerwaiting + summary: Pod container waiting longer than 1 hour + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) > 0 + for: {{ dig "KubeContainerWaiting" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeContainerWaiting" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetNotScheduled | default false) }} + - alert: KubeDaemonSetNotScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetnotscheduled + summary: DaemonSet pods are not scheduled. + expr: |- + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + - + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetNotScheduled" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetNotScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetMisScheduled | default false) }} + - alert: KubeDaemonSetMisScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetmisscheduled + summary: DaemonSet pods are misscheduled. + expr: kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetMisScheduled" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetMisScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobNotCompleted | default false) }} + - alert: KubeJobNotCompleted + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than {{`{{`}} "43200" | humanizeDuration {{`}}`}} to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobnotcompleted + summary: Job did not complete in time + expr: |- + time() - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, job_name, cluster) (kube_job_status_start_time{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + and + kube_job_status_active{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0) > 43200 + labels: + severity: {{ dig "KubeJobNotCompleted" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobFailed | default false) }} + - alert: KubeJobFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. Removing failed job after investigation should clear this alert. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobfailed + summary: Job failed to complete. + expr: kube_job_failed{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeJobFailed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeJobFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaReplicasMismatch | default false) }} + - alert: KubeHpaReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpareplicasmismatch + summary: HPA has not matched desired number of replicas. + expr: |- + (kube_horizontalpodautoscaler_status_desired_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_horizontalpodautoscaler_spec_min_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + < + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + changes(kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 + for: {{ dig "KubeHpaReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaMaxedOut | default false) }} + - alert: KubeHpaMaxedOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has been running at max replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpamaxedout + summary: HPA is running at max replicas + expr: |- + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + == + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeHpaMaxedOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaMaxedOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml new file mode 100644 index 0000000000..1d32f9bbad --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -0,0 +1,282 @@ +{{- /* +Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesResources }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-resources" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-resources + rules: +{{- if not (.Values.defaultRules.disabled.KubeCPUOvercommit | default false) }} + - alert: KubeCPUOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Pods by {{`{{`}} $value {{`}}`}} CPU shares and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(namespace_cpu:kube_pod_container_resource_requests:sum{job="{{ $kubeStateMetricsJob }}",}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeCPUOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryOvercommit | default false) }} + - alert: KubeMemoryOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeMemoryOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeCPUQuotaOvercommit | default false) }} + - alert: KubeCPUQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuquotaovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(cpu|requests.cpu)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="cpu", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeCPUQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryQuotaOvercommit | default false) }} + - alert: KubeMemoryQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryquotaovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(memory|requests.memory)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeMemoryQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaAlmostFull | default false) }} + - alert: KubeQuotaAlmostFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaalmostfull + summary: Namespace quota is going to be full. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 0.9 < 1 + for: {{ dig "KubeQuotaAlmostFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaAlmostFull" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaFullyUsed | default false) }} + - alert: KubeQuotaFullyUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotafullyused + summary: Namespace quota is fully used. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + == 1 + for: {{ dig "KubeQuotaFullyUsed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaFullyUsed" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaExceeded | default false) }} + - alert: KubeQuotaExceeded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaexceeded + summary: Namespace quota has exceeded the limits. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 1 + for: {{ dig "KubeQuotaExceeded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaExceeded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.CPUThrottlingHigh | default false) }} + - alert: CPUThrottlingHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.pod {{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/cputhrottlinghigh + summary: Processes experience elevated CPU throttling. + expr: |- + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + / + sum(increase(container_cpu_cfs_periods_total{}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + > ( 25 / 100 ) + for: {{ dig "CPUThrottlingHigh" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "CPUThrottlingHigh" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml new file mode 100644 index 0000000000..b988445653 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -0,0 +1,216 @@ +{{- /* +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-storage + rules: +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + < 0.03 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} only has {{`{{`}} $value | humanizePercentage {{`}}`}} free inodes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to run out of inodes within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} of its inodes are free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeErrors | default false) }} + - alert: KubePersistentVolumeErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeerrors + summary: PersistentVolume is having issues with provisioning. + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="{{ $kubeStateMetricsJob }}"} > 0 + for: {{ dig "KubePersistentVolumeErrors" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml new file mode 100644 index 0000000000..af34a23f88 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -0,0 +1,193 @@ +{{- /* +Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-apiserver" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-apiserver + rules: +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIErrors | default false) }} + - alert: KubeAggregatedAPIErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapierrors + summary: Kubernetes aggregated API has reported errors. + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4 + labels: + severity: {{ dig "KubeAggregatedAPIErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIDown | default false) }} + - alert: KubeAggregatedAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapidown + summary: Kubernetes aggregated API is down. + expr: (1 - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85 + for: {{ dig "KubeAggregatedAPIDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAggregatedAPIDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if .Values.kubeApiServer.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeAPIDown | default false) }} + - alert: KubeAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: KubeAPI has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapidown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="apiserver"} == 1) + for: {{ dig "KubeAPIDown" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPIDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPITerminatedRequests | default false) }} + - alert: KubeAPITerminatedRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapiterminatedrequests + summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 + for: {{ dig "KubeAPITerminatedRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPITerminatedRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml new file mode 100644 index 0000000000..205bd59800 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeControllerManager }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-controller-manager" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-controller-manager + rules: +{{- if not (.Values.defaultRules.disabled.KubeControllerManagerDown | default false) }} + - alert: KubeControllerManagerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager | indent 8 }} +{{- end }} + description: KubeControllerManager has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontrollermanagerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeControllerManager.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeControllerManagerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} + diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml new file mode 100644 index 0000000000..66b1d62001 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml @@ -0,0 +1,56 @@ +{{- /* +Generated from 'kubernetes-system-kube-proxy' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeProxy }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kube-proxy" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kube-proxy + rules: +{{- if not (.Values.defaultRules.disabled.KubeProxyDown | default false) }} + - alert: KubeProxyDown + annotations: + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + description: KubeProxy has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeproxydown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeProxy.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeProxyDown" "labelsSeverity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml new file mode 100644 index 0000000000..2a55676735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -0,0 +1,379 @@ +{{- /* +Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kubelet" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kubelet + rules: +{{- if not (.Values.defaultRules.disabled.KubeNodeNotReady | default false) }} + - alert: KubeNodeNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than 15 minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodenotready + summary: Node is not ready. + expr: kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",condition="Ready",status="true"} == 0 + for: {{ dig "KubeNodeNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeUnreachable | default false) }} + - alert: KubeNodeUnreachable + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} is unreachable and some workloads may be rescheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodeunreachable + summary: Node is unreachable. + expr: (kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1 + for: {{ dig "KubeNodeUnreachable" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeUnreachable" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletTooManyPods | default false) }} + - alert: KubeletTooManyPods + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubelettoomanypods + summary: Kubelet is running at capacity. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + (kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}",phase="Running"} == 1) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) (1, kube_pod_info{job="{{ $kubeStateMetricsJob }}"}) + ) + / + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + kube_node_status_capacity{job="{{ $kubeStateMetricsJob }}",resource="pods"} != 1 + ) > 0.95 + for: {{ dig "KubeletTooManyPods" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletTooManyPods" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeReadinessFlapping | default false) }} + - alert: KubeNodeReadinessFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodereadinessflapping + summary: Node readiness status is flapping. + expr: sum(changes(kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",status="true",condition="Ready"}[15m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) > 2 + for: {{ dig "KubeNodeReadinessFlapping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeReadinessFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPlegDurationHigh | default false) }} + - alert: KubeletPlegDurationHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletplegdurationhigh + summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist. + expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10 + for: {{ dig "KubeletPlegDurationHigh" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletPlegDurationHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPodStartUpLatencyHigh | default false) }} + - alert: KubeletPodStartUpLatencyHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletpodstartuplatencyhigh + summary: Kubelet Pod startup latency is too high. + expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} > 60 + for: 15m + labels: + severity: {{ dig "KubeletPodStartUpLatencyHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateRenewalErrors | default false) }} + - alert: KubeletClientCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its client certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificaterenewalerrors + summary: Kubelet has failed to renew its client certificate. + expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletClientCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletClientCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateRenewalErrors | default false) }} + - alert: KubeletServerCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its server certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificaterenewalerrors + summary: Kubelet has failed to renew its server certificate. + expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletServerCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletServerCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if (include "exporter.kubelet.enabled" .)}} +{{- if not (.Values.defaultRules.disabled.KubeletDown | default false) }} + - alert: KubeletDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} == 1) + for: 15m + labels: + severity: {{ dig "KubeletDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml new file mode 100644 index 0000000000..9890b1c959 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -0,0 +1,54 @@ +{{- /* +Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-scheduler" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-scheduler + rules: +{{- if .Values.kubeScheduler.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeSchedulerDown | default false) }} + - alert: KubeSchedulerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting | indent 8 }} +{{- end }} + description: KubeScheduler has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeschedulerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeScheduler.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeSchedulerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml new file mode 100644 index 0000000000..621326d0ad --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -0,0 +1,87 @@ +{{- /* +Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system + rules: +{{- if not (.Values.defaultRules.disabled.KubeVersionMismatch | default false) }} + - alert: KubeVersionMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeversionmismatch + summary: Different semantic versions of Kubernetes components running. + expr: count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 + for: {{ dig "KubeVersionMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeVersionMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientErrors | default false) }} + - alert: KubeClientErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} $value | humanizePercentage {{`}}`}} errors.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclienterrors + summary: Kubernetes API server client is experiencing errors. + expr: |- + (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace) + / + sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace)) + > 0.01 + for: {{ dig "KubeClientErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml new file mode 100644 index 0000000000..5d4711ae08 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml @@ -0,0 +1,188 @@ +{{- /* +Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter.rules + rules: + - expr: |- + count without (cpu, mode) ( + node_cpu_seconds_total{job="node-exporter",mode="idle"} + ) + record: instance:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg without (cpu) ( + sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m])) + ) + record: instance:node_cpu_utilisation:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + ( + node_load1{job="node-exporter"} + / + instance:node_num_cpu:sum{job="node-exporter"} + ) + record: instance:node_load1_per_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + node_memory_MemAvailable_bytes{job="node-exporter"} + or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + + node_memory_Cached_bytes{job="node-exporter"} + + + node_memory_MemFree_bytes{job="node-exporter"} + + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) + / + node_memory_MemTotal_bytes{job="node-exporter"} + ) + record: instance:node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) + record: instance:node_vmstat_pgmajfault:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_weighted_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml new file mode 100644 index 0000000000..14738cedfa --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml @@ -0,0 +1,801 @@ +{{- /* +Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter + rules: +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 24 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 4 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 5% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 3% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 24 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 4 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 5% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 3% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkReceiveErrs | default false) }} + - alert: NodeNetworkReceiveErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworkreceiveerrs + summary: Network interface is reporting many receive errors. + expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkReceiveErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkReceiveErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkTransmitErrs | default false) }} + - alert: NodeNetworkTransmitErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworktransmiterrs + summary: Network interface is reporting many transmit errors. + expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkTransmitErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkTransmitErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeHighNumberConntrackEntriesUsed | default false) }} + - alert: NodeHighNumberConntrackEntriesUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodehighnumberconntrackentriesused + summary: Number of conntrack are getting close to the limit. + expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75 + labels: + severity: {{ dig "NodeHighNumberConntrackEntriesUsed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeTextFileCollectorScrapeError | default false) }} + - alert: NodeTextFileCollectorScrapeError + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodetextfilecollectorscrapeerror + summary: Node Exporter text file collector failed to scrape. + expr: node_textfile_scrape_error{job="node-exporter"} == 1 + labels: + severity: {{ dig "NodeTextFileCollectorScrapeError" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockSkewDetected | default false) }} + - alert: NodeClockSkewDetected + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclockskewdetected + summary: Clock skew detected. + expr: |- + ( + node_timex_offset_seconds{job="node-exporter"} > 0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0 + ) + or + ( + node_timex_offset_seconds{job="node-exporter"} < -0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0 + ) + for: {{ dig "NodeClockSkewDetected" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockSkewDetected" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockNotSynchronising | default false) }} + - alert: NodeClockNotSynchronising + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclocknotsynchronising + summary: Clock not synchronising. + expr: |- + min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0 + and + node_timex_maxerror_seconds{job="node-exporter"} >= 16 + for: {{ dig "NodeClockNotSynchronising" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockNotSynchronising" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDegraded | default false) }} + - alert: NodeRAIDDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: RAID array '{{`{{`}} $labels.device {{`}}`}}' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddegraded + summary: RAID Array is degraded. + expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0 + for: {{ dig "NodeRAIDDegraded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeRAIDDegraded" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDiskFailure | default false) }} + - alert: NodeRAIDDiskFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddiskfailure + summary: Failed device in RAID array. + expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0 + labels: + severity: {{ dig "NodeRAIDDiskFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeCPUHighUsage | default false) }} + - alert: NodeCPUHighUsage + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodecpuhighusage + summary: High CPU usage. + expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!="idle"}[2m]))) * 100 > 90 + for: {{ dig "NodeCPUHighUsage" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeCPUHighUsage" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemSaturation | default false) }} + - alert: NodeSystemSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This might indicate this instance resources saturation and can cause it becoming unresponsive. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemsaturation + summary: System saturated, load per core is very high. + expr: |- + node_load1{job="node-exporter"} + / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2 + for: {{ dig "NodeSystemSaturation" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryMajorPagesFaults | default false) }} + - alert: NodeMemoryMajorPagesFaults + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + Please check that there is enough memory available at this instance. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememorymajorpagesfaults + summary: Memory major page faults are occurring at very high rate. + expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500 + for: {{ dig "NodeMemoryMajorPagesFaults" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryMajorPagesFaults" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryHighUtilization | default false) }} + - alert: NodeMemoryHighUtilization + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememoryhighutilization + summary: Host is running out of memory. + expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90 + for: {{ dig "NodeMemoryHighUtilization" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryHighUtilization" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeDiskIOSaturation | default false) }} + - alert: NodeDiskIOSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 30 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This symptom might indicate disk saturation. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodediskiosaturation + summary: Disk IO queue is high. + expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10 + for: {{ dig "NodeDiskIOSaturation" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeDiskIOSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemdServiceFailed | default false) }} + - alert: NodeSystemdServiceFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemdservicefailed + summary: Systemd service has entered failed state. + expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1 + for: {{ dig "NodeSystemdServiceFailed" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemdServiceFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeBondingDegraded | default false) }} + - alert: NodeBondingDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodebondingdegraded + summary: Bonding interface is degraded + expr: (node_bonding_slaves - node_bonding_active) != 0 + for: {{ dig "NodeBondingDegraded" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeBondingDegraded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml new file mode 100644 index 0000000000..8dc60ef66b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.network }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-network" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-network + rules: +{{- if not (.Values.defaultRules.disabled.NodeNetworkInterfaceFlapping | default false) }} + - alert: NodeNetworkInterfaceFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.network }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.network | indent 8 }} +{{- end }} + description: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing its up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/nodenetworkinterfaceflapping + summary: Network interface is often changing its status + expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 + for: {{ dig "NodeNetworkInterfaceFlapping" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkInterfaceFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml new file mode 100644 index 0000000000..e2911b905e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml @@ -0,0 +1,109 @@ +{{- /* +Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.node }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node.rules + rules: + - expr: |- + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node, namespace, pod) ( + label_replace(kube_pod_info{job="{{ $kubeStateMetricsJob }}",node!=""}, "pod", "$1", "pod", "(.*)") + )) + record: 'node_namespace_pod:kube_pod_info:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + node_cpu_seconds_total{mode="idle",job="node-exporter"} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, node_namespace_pod:kube_pod_info:) + ) + record: node:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum( + node_memory_MemAvailable_bytes{job="node-exporter"} or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + node_memory_Cached_bytes{job="node-exporter"} + + node_memory_MemFree_bytes{job="node-exporter"} + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + record: :node_memory_MemAvailable_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + sum without (mode) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m]) + ) + ) + record: node:node_cpu_utilization:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + node:node_cpu_utilization:ratio_rate5m + ) + record: cluster:node_cpu:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml new file mode 100644 index 0000000000..1f288dffe3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml @@ -0,0 +1,253 @@ +{{- /* +Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheusOperator }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus-operator" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus-operator + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorListErrors | default false) }} + - alert: PrometheusOperatorListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing List operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorlisterrors + summary: Errors while performing list operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m]))) > 0.4 + for: {{ dig "PrometheusOperatorListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorListErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorWatchErrors | default false) }} + - alert: PrometheusOperatorWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing watch operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorwatcherrors + summary: Errors while performing watch operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.4 + for: {{ dig "PrometheusOperatorWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorWatchErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorSyncFailed | default false) }} + - alert: PrometheusOperatorSyncFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Controller {{`{{`}} $labels.controller {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} namespace fails to reconcile {{`{{`}} $value {{`}}`}} objects. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorsyncfailed + summary: Last controller reconciliation failed + expr: min_over_time(prometheus_operator_syncs{status="failed",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorSyncFailed" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorSyncFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorReconcileErrors | default false) }} + - alert: PrometheusOperatorReconcileErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of reconciling operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorreconcileerrors + summary: Errors while reconciling objects. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorReconcileErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorReconcileErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorStatusUpdateErrors | default false) }} + - alert: PrometheusOperatorStatusUpdateErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of status update operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorstatusupdateerrors + summary: Errors while updating objects status. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorStatusUpdateErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorStatusUpdateErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNodeLookupErrors | default false) }} + - alert: PrometheusOperatorNodeLookupErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while reconciling Prometheus in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornodelookuperrors + summary: Errors while reconciling Prometheus. + expr: rate(prometheus_operator_node_address_lookup_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: {{ dig "PrometheusOperatorNodeLookupErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNodeLookupErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNotReady | default false) }} + - alert: PrometheusOperatorNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace isn't ready to reconcile {{`{{`}} $labels.controller {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornotready + summary: Prometheus operator not ready + expr: min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) == 0) + for: {{ dig "PrometheusOperatorNotReady" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorRejectedResources | default false) }} + - alert: PrometheusOperatorRejectedResources + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace rejected {{`{{`}} printf "%0.0f" $value {{`}}`}} {{`{{`}} $labels.controller {{`}}`}}/{{`{{`}} $labels.resource {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorrejectedresources + summary: Resources rejected by Prometheus operator + expr: min_over_time(prometheus_operator_managed_resources{state="rejected",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorRejectedResources" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorRejectedResources" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml new file mode 100644 index 0000000000..9dfeb1f9db --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml @@ -0,0 +1,707 @@ +{{- /* +Generated from 'prometheus' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} +{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusBadConfig | default false) }} + - alert: PrometheusBadConfig + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to reload its configuration. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusbadconfig + summary: Failed Prometheus configuration reload. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "PrometheusBadConfig" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusBadConfig" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusSDRefreshFailure | default false) }} + - alert: PrometheusSDRefreshFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to refresh SD with mechanism {{`{{`}}$labels.mechanism{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheussdrefreshfailure + summary: Failed Prometheus SD refresh. + expr: increase(prometheus_sd_refresh_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[10m]) > 0 + for: {{ dig "PrometheusSDRefreshFailure" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusSDRefreshFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotificationQueueRunningFull | default false) }} + - alert: PrometheusNotificationQueueRunningFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Alert notification queue of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is running full. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotificationqueuerunningfull + summary: Prometheus alert notification queue predicted to run full in less than 30m. + expr: |- + # Without min_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) + > + min_over_time(prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusNotificationQueueRunningFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotificationQueueRunningFull" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToSomeAlertmanagers | default false) }} + - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.alertmanager{{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstosomealertmanagers + summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager. + expr: |- + ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + * 100 + > 1 + for: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotConnectedToAlertmanagers | default false) }} + - alert: PrometheusNotConnectedToAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not connected to any Alertmanagers. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotconnectedtoalertmanagers + summary: Prometheus is not connected to any Alertmanagers. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) < 1 + for: {{ dig "PrometheusNotConnectedToAlertmanagers" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotConnectedToAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBReloadsFailing | default false) }} + - alert: PrometheusTSDBReloadsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} reload failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbreloadsfailing + summary: Prometheus has issues reloading blocks from disk. + expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBReloadsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBReloadsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBCompactionsFailing | default false) }} + - alert: PrometheusTSDBCompactionsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbcompactionsfailing + summary: Prometheus has issues compacting blocks. + expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBCompactionsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBCompactionsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotIngestingSamples | default false) }} + - alert: PrometheusNotIngestingSamples + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not ingesting samples. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotingestingsamples + summary: Prometheus is not ingesting samples. + expr: |- + ( + sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) <= 0 + and + ( + sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + or + sum without(rule_group) (prometheus_rule_group_rules{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + ) + ) + for: {{ dig "PrometheusNotIngestingSamples" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotIngestingSamples" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusDuplicateTimestamps | default false) }} + - alert: PrometheusDuplicateTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with different values but duplicated timestamp. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusduplicatetimestamps + summary: Prometheus is dropping samples with duplicate timestamps. + expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusDuplicateTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusDuplicateTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOutOfOrderTimestamps | default false) }} + - alert: PrometheusOutOfOrderTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with timestamps arriving out of order. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusoutofordertimestamps + summary: Prometheus drops samples with out-of-order timestamps. + expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOutOfOrderTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOutOfOrderTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteStorageFailures | default false) }} + - alert: PrometheusRemoteStorageFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotestoragefailures + summary: Prometheus fails to send samples to remote storage. + expr: |- + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + / + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + + + (rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + ) + ) + * 100 + > 1 + for: {{ dig "PrometheusRemoteStorageFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteStorageFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteBehind | default false) }} + - alert: PrometheusRemoteWriteBehind + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritebehind + summary: Prometheus remote write is behind. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + - ignoring(remote_name, url) group_right + max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 120 + for: {{ dig "PrometheusRemoteWriteBehind" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteBehind" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteDesiredShards | default false) }} + - alert: PrometheusRemoteWriteDesiredShards + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritedesiredshards + summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_shards_desired{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + > + max_over_time(prometheus_remote_storage_shards_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusRemoteWriteDesiredShards" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteDesiredShards" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRuleFailures | default false) }} + - alert: PrometheusRuleFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to evaluate {{`{{`}} printf "%.0f" $value {{`}}`}} rules in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusrulefailures + summary: Prometheus is failing rule evaluations. + expr: increase(prometheus_rule_evaluation_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusRuleFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRuleFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusMissingRuleEvaluations | default false) }} + - alert: PrometheusMissingRuleEvaluations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has missed {{`{{`}} printf "%.0f" $value {{`}}`}} rule group evaluations in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusmissingruleevaluations + summary: Prometheus is missing rule evaluations due to slow rule group evaluation. + expr: increase(prometheus_rule_group_iterations_missed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusMissingRuleEvaluations" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusMissingRuleEvaluations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetLimitHit | default false) }} + - alert: PrometheusTargetLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because the number of targets exceeded the configured target_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetlimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit. + expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusTargetLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusLabelLimitHit | default false) }} + - alert: PrometheusLabelLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuslabellimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit. + expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusLabelLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusLabelLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeBodySizeLimitHit | default false) }} + - alert: PrometheusScrapeBodySizeLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured body_size_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapebodysizelimithit + summary: Prometheus has dropped some targets that exceeded body size limit. + expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeBodySizeLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeBodySizeLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeSampleLimitHit | default false) }} + - alert: PrometheusScrapeSampleLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured sample_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapesamplelimithit + summary: Prometheus has failed scrapes that have exceeded the configured sample limit. + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeSampleLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeSampleLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetSyncFailure | default false) }} + - alert: PrometheusTargetSyncFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.0f" $value {{`}}`}} targets in Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} have failed to sync because invalid configuration was supplied.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetsyncfailure + summary: Prometheus has failed to sync targets. + expr: increase(prometheus_target_sync_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[30m]) > 0 + for: {{ dig "PrometheusTargetSyncFailure" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetSyncFailure" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusHighQueryLoad | default false) }} + - alert: PrometheusHighQueryLoad + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} query API has less than 20% available capacity in its query engine for the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheushighqueryload + summary: Prometheus is reaching its maximum capacity serving concurrent requests. + expr: avg_over_time(prometheus_engine_queries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.8 + for: {{ dig "PrometheusHighQueryLoad" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusHighQueryLoad" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToAnyAlertmanager | default false) }} + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstoanyalertmanager + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without (alertmanager) ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + ) + * 100 + > 3 + for: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml new file mode 100644 index 0000000000..7c25553861 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml @@ -0,0 +1,301 @@ +{{- /* +Generated from 'windows.node.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.node.rules + rules: + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + windows_system_system_up_time{job="windows-exporter"} + ) + record: node:windows_node:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, core) ( + windows_cpu_time_total{job="windows-exporter"} + )) + record: node:windows_node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m])) + record: :windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m]) + ) + record: node:windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"}) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: ':windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"} + windows_memory_cache_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemFreeCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: (windows_memory_cache_bytes{job="windows-exporter"} + windows_memory_modified_page_list_bytes{job="windows-exporter"} + windows_memory_standby_cache_core_bytes{job="windows-exporter"} + windows_memory_standby_cache_normal_priority_bytes{job="windows-exporter"} + windows_memory_standby_cache_reserve_bytes{job="windows-exporter"}) + record: node:windows_node_memory_totalCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemTotal_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (windows_memory_available_bytes{job="windows-exporter"}) + ) + record: node:windows_node_memory_bytes_available:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + windows_os_visible_memory_bytes{job="windows-exporter"} + ) + record: node:windows_node_memory_bytes_total:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + (node:windows_node_memory_bytes_total:sum - node:windows_node_memory_bytes_available:sum) + / + scalar(sum(node:windows_node_memory_bytes_total:sum)) + record: node:windows_node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - (node:windows_node_memory_bytes_available:sum / node:windows_node_memory_bytes_total:sum) + record: 'node:windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: irate(windows_memory_swap_page_operations_total{job="windows-exporter"}[5m]) + record: node:windows_node_memory_swap_io_pages:irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m]) + ) + record: :windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,instance,volume)( + (windows_logical_disk_size_bytes{job="windows-exporter"} + - windows_logical_disk_free_bytes{job="windows-exporter"}) + / windows_logical_disk_size_bytes{job="windows-exporter"} + ) + record: 'node:windows_node_filesystem_usage:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, volume) (windows_logical_disk_free_bytes{job="windows-exporter"} / windows_logical_disk_size_bytes{job="windows-exporter"}) + record: 'node:windows_node_filesystem_avail:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + record: :windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m])) + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + record: :windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m]) + + irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml new file mode 100644 index 0000000000..86340b5c05 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml @@ -0,0 +1,158 @@ +{{- /* +Generated from 'windows.pod.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.pod.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.pod.rules + rules: + - expr: windows_container_available{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_pod_container_available + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_cpu_usage_seconds_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_total_runtime + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_commit_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_memory_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_private_working_set_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_private_working_set_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_receive_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_received_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_transmit_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_transmitted_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + rate(windows_container_total_runtime{}[5m]) + ) + record: namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/secret.yaml new file mode 100644 index 0000000000..e4a1e73c7b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/secret.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.objectStorageConfig}} +{{- if and .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret (not .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + object-storage-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/service.yaml new file mode 100644 index 0000000000..d61b9d6ef3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/service.yaml @@ -0,0 +1,80 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + self-monitor: {{ .Values.prometheus.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.service.labels }} +{{ toYaml .Values.prometheus.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheus.service.clusterIP }} + clusterIP: {{ .Values.prometheus.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.service.nodePort }} + {{- end }} + port: {{ .Values.prometheus.service.port }} + targetPort: {{ .Values.prometheus.service.targetPort }} + - name: reloader-web + {{- if semverCompare "> 1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: {{ .Values.prometheus.service.reloaderWebPort }} + targetPort: reloader-web + {{- if .Values.prometheus.thanosIngress.enabled }} + - name: grpc + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosIngress.nodePort }} + {{- end }} + port: {{ .Values.prometheus.thanosIngress.servicePort }} + targetPort: {{ .Values.prometheus.thanosIngress.servicePort }} + {{- end }} +{{- if .Values.prometheus.service.additionalPorts }} +{{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} +{{- end }} + publishNotReadyAddresses: {{ .Values.prometheus.service.publishNotReadyAddresses }} + selector: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- if .Values.prometheus.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.prometheus.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.prometheus.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.prometheus.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml new file mode 100644 index 0000000000..15b89c8c23 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-discovery + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosService.labels }} +{{ toYaml .Values.prometheus.thanosService.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosService.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosService.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosService.type }} + clusterIP: {{ .Values.prometheus.thanosService.clusterIP }} +{{- if ne .Values.prometheus.thanosService.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosService.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosService.portName }} + port: {{ .Values.prometheus.thanosService.port }} + targetPort: {{ .Values.prometheus.thanosService.targetPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosService.httpPortName }} + port: {{ .Values.prometheus.thanosService.httpPort }} + targetPort: {{ .Values.prometheus.thanosService.targetHttpPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml new file mode 100644 index 0000000000..453eed7f1b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosServiceExternal.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-external + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosServiceExternal.labels }} +{{ toYaml .Values.prometheus.thanosServiceExternal.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosServiceExternal.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosServiceExternal.type }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.thanosServiceExternal.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.thanosServiceExternal.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosServiceExternal.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosServiceExternal.portName }} + port: {{ .Values.prometheus.thanosServiceExternal.port }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosServiceExternal.httpPortName }} + port: {{ .Values.prometheus.thanosServiceExternal.httpPort }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetHttpPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceaccount.yaml new file mode 100644 index 0000000000..e97b989bbd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.prometheus.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.prometheus.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitor.yaml new file mode 100644 index 0000000000..a36f3e33ca --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitor.yaml @@ -0,0 +1,97 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.serviceMonitor.bearerTokenFile }} + {{- end }} + path: "{{ trimSuffix "/" .Values.prometheus.prometheusSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + - port: reloader-web + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "/metrics" + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.prometheus.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.prometheus.serviceMonitor.interval .interval }} + interval: {{ default $.Values.prometheus.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.prometheus.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.prometheus.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml new file mode 100644 index 0000000000..0f70aabb58 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.prometheus.thanosService.enabled .Values.prometheus.thanosServiceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-sidecar + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-sidecar +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.thanosServiceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.thanosServiceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.thanosService.httpPortName }} + {{- if .Values.prometheus.thanosServiceMonitor.interval }} + interval: {{ .Values.prometheus.thanosServiceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.scheme }} + scheme: {{ .Values.prometheus.thanosServiceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.prometheus.thanosServiceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + {{- end }} + path: "/metrics" + metricRelabelings: + {{- if .Values.prometheus.thanosServiceMonitor.metricRelabelings}} + {{ tpl (toYaml .Values.prometheus.thanosServiceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheus.thanosServiceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.thanosServiceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitors.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitors.yaml new file mode 100644 index 0000000000..a7a301babc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitors.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalServiceMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalServiceMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + endpoints: +{{ toYaml .endpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .targetLabels }} + targetLabels: +{{ toYaml .targetLabels | indent 8 }} + {{- end }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .metricRelabelings }} + metricRelabelings: +{{ toYaml .metricRelabelings | indent 8 }} + {{- end }} + {{- if .relabelings }} + relabelings: +{{ toYaml .relabelings | indent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceperreplica.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceperreplica.yaml new file mode 100644 index 0000000000..4bc7f7b869 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceperreplica.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $serviceValues := .Values.prometheus.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.prometheus.prometheusSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + {{- if $.Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + statefulset.kubernetes.io/pod-name: prom-agent-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- else }} + app.kubernetes.io/name: prometheus + statefulset.kubernetes.io/pod-name: prometheus-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" $ }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/clusterrole.yaml new file mode 100644 index 0000000000..56ca9f5eae --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/clusterrole.yaml @@ -0,0 +1,135 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-admin + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-edit + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-ui-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" +{{- if .Values.grafana.enabled }} + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" +{{- end }} + verbs: + - 'get' + - 'create' +- apiGroups: + - "" + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- if .Values.grafana.enabled }} + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +{{- end }} + resources: + - endpoints + verbs: + - list +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/config-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/config-role.yaml new file mode 100644 index 0000000000..f48ffc827e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/config-role.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-admin + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-edit + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-view + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml new file mode 100644 index 0000000000..d2f81976a2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create .Values.grafana.enabled }} +{{- if .Values.grafana.defaultDashboardsEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-admin + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-edit + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-view + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml new file mode 100644 index 0000000000..7b51a0bf7a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.ingressNginx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "ingress-nginx" | trunc 63 | trimSuffix "-" }} + {{- if .Values.grafana.sidecar.dashboards.annotations }} + annotations: {{ toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} + {{- end }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/ingress-nginx/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml new file mode 100644 index 0000000000..d73b257451 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-cluster + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/cluster/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml new file mode 100644 index 0000000000..8865efa932 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-home + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/home/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml new file mode 100644 index 0000000000..9b05cea2e8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-fleet-dashboards + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/fleet/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml new file mode 100644 index 0000000000..2afae10ef7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml @@ -0,0 +1,31 @@ +{{- $files := (.Files.Glob "files/rancher/k8s/*").AsConfig }} +{{- $filesDict := (fromYaml $files) }} +{{- if not (include "exporter.kubeEtcd.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-etcd-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-etcd.json") -}} +{{- end }} +{{- if not (include "exporter.kubeControllerManager.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-k8s-components-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-k8s-components.json") -}} +{{- else }} +{{- $_ := (set $filesDict "rancher-k8s-components-nodes.json" (get $filesDict "rancher-k8s-components-nodes.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- $_ := (set $filesDict "rancher-k8s-components.json" (get $filesDict "rancher-k8s-components.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- end }} +{{ $files = (toYaml $filesDict) }} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-k8s + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ $files | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml new file mode 100644 index 0000000000..172c36e9d1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-nodes + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/nodes/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml new file mode 100644 index 0000000000..19836ec4e4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml @@ -0,0 +1,18 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.rancherMonitoring.enabled $selector }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-performance-debugging + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/performance/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml new file mode 100644 index 0000000000..940f18869b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-pods + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/pods/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml new file mode 100644 index 0000000000..d146dacdd0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-workloads + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/workloads/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml new file mode 100644 index 0000000000..90d24c2061 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml @@ -0,0 +1,53 @@ +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-fleet-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: fleet + selector: + matchLabels: + app: fleet-controller +{{- end }} +--- +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-gitops-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: gitops + selector: + matchLabels: + app: gitjob +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml new file mode 100644 index 0000000000..53a9ad6897 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml @@ -0,0 +1,27 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + jobLabel: ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: {{ .Values.ingressNginx.namespace }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.ingressNginx.service.port }} + protocol: TCP + targetPort: {{ .Values.ingressNginx.service.targetPort }} + selector: + {{- if .Values.ingressNginx.service.selector }} +{{ toYaml .Values.ingressNginx.service.selector | indent 4 }} + {{- else }} + app: ingress-nginx + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml new file mode 100644 index 0000000000..b0f92e63b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml @@ -0,0 +1,49 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + namespace: {{ .Values.ingressNginx.namespace }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Values.ingressNginx.namespace }} + endpoints: + - port: http-metrics + {{- if .Values.ingressNginx.serviceMonitor.interval}} + interval: {{ .Values.ingressNginx.serviceMonitor.interval }} + {{- end }} + {{- if .Values.ingressNginx.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.ingressNginx.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.ingressNginx.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.ingressNginx.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.ingressNginx.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.ingressNginx.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml new file mode 100644 index 0000000000..1fba8f23f7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.rancherMonitoring.enabled $selector }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: rancher + namespace: cattle-system +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + port: http + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + serverName: rancher + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: rancher +{{- if .Values.rancherMonitoring.namespaceSelector }} + namespaceSelector: {{ .Values.rancherMonitoring.namespaceSelector | toYaml | nindent 4 }} +{{- end }} + selector: {{ include "rancher.serviceMonitor.selector" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +rules: +- apiGroups: + - management.cattle.io + resources: + - ranchermetrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/hardened.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/hardened.yaml new file mode 100644 index 0000000000..f9a66151ee --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/hardened.yaml @@ -0,0 +1,147 @@ +{{- $namespaces := dict "_0" .Release.Namespace -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) -}} +{{- $_ := set $namespaces "_1" .Values.grafana.defaultDashboards.namespace -}} +{{- end -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa + spec: + serviceAccountName: {{ .Chart.Name }}-patch-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + {{- range $_, $ns := $namespaces }} + - name: patch-sa-{{ $ns }} + image: {{ template "system_default_registry" $ }}{{ $.Values.global.kubectl.repository }}:{{ $.Values.global.kubectl.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", "{{ $ns }}"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +rules: +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: ['get', 'patch'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-patch-sa +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-patch-sa +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- range $_, $ns := $namespaces }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ $ns }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +{{- end }} +{{- end }} +--- +{{- if .Values.hardened.k3s.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rancher-monitoring-coredns-allow-all + namespace: kube-system +spec: + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + podSelector: + matchLabels: + k8s-app: kube-dns +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml new file mode 100644 index 0000000000..53cb898214 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "0" +data: +{{ (.Files.Glob "files/upgrade/scripts/*").AsConfig | indent 2 }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml new file mode 100644 index 0000000000..8f2771740c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml @@ -0,0 +1,46 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "2" +spec: + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + spec: + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + securityContext: + runAsNonRoot: false + runAsUser: 0 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + - name: run-scripts + image: {{ template "system_default_registry" . }}{{ .Values.upgrade.image.repository }}:{{ .Values.upgrade.image.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: + - /bin/sh + - -c + - > + for s in $(find /etc/scripts -type f); do + echo "Running $s..."; + cat $s | bash + done; + volumeMounts: + - name: upgrade + mountPath: /etc/scripts + volumes: + - name: upgrade + configMap: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml new file mode 100644 index 0000000000..e929a19925 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml @@ -0,0 +1,131 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded + "helm.sh/hook-weight": "1" +rules: +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - 'list' + - 'delete' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/extrasecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/extrasecret.yaml new file mode 100644 index 0000000000..587fca2dca --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.thanosRuler.extraSecret.data -}} +{{- $secretName := printf "%s-extra" (include "kube-prometheus-stack.thanosRuler.name" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.thanosRuler.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.extraSecret.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.thanosRuler.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ingress.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ingress.yaml new file mode 100644 index 0000000000..e245ad448e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.ingress.enabled }} +{{- $pathType := .Values.thanosRuler.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := include "kube-prometheus-stack.thanosRuler.name" . }} +{{- $servicePort := .Values.thanosRuler.service.port -}} +{{- $routePrefix := list .Values.thanosRuler.thanosRulerSpec.routePrefix }} +{{- $paths := .Values.thanosRuler.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.thanosRuler.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- if .Values.thanosRuler.ingress.labels }} +{{ toYaml .Values.thanosRuler.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.thanosRuler.ingress.ingressClassName }} + ingressClassName: {{ .Values.thanosRuler.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.thanosRuler.ingress.hosts }} + {{- range $host := .Values.thanosRuler.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.thanosRuler.ingress.tls }} + tls: +{{ tpl (toYaml .Values.thanosRuler.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml new file mode 100644 index 0000000000..83e54edf9b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.thanosRuler.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.thanosRuler.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ruler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ruler.yaml new file mode 100644 index 0000000000..b281221563 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ruler.yaml @@ -0,0 +1,189 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ThanosRuler +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.thanosRulerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.thanosRuler.thanosRulerSpec.image.registry -}} + {{- if and .Values.thanosRuler.thanosRulerSpec.image.tag .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}" + {{- end }} + {{- if .Values.thanosRuler.thanosRulerSpec.image.sha }} + sha: {{ .Values.thanosRuler.thanosRulerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.thanosRuler.thanosRulerSpec.replicas }} + listenLocal: {{ .Values.thanosRuler.thanosRulerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} +{{- if .Values.thanosRuler.thanosRulerSpec.externalPrefix }} + externalPrefix: "{{ tpl .Values.thanosRuler.thanosRulerSpec.externalPrefix . }}" +{{- else if and .Values.thanosRuler.ingress.enabled .Values.thanosRuler.ingress.hosts }} + externalPrefix: "http://{{ tpl (index .Values.thanosRuler.ingress.hosts 0) . }}{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- else }} + externalPrefix: http://{{ template "kube-prometheus-stack.thanosRuler.name" . }}.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.thanosRuler.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.additionalArgs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.nodeSelector }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.thanosRuler.thanosRulerSpec.paused }} + logFormat: {{ .Values.thanosRuler.thanosRulerSpec.logFormat | quote }} + logLevel: {{ .Values.thanosRuler.thanosRulerSpec.logLevel | quote }} + retention: {{ .Values.thanosRuler.thanosRulerSpec.retention | quote }} +{{- if .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} + evaluationInterval: {{ .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleSelector | indent 4) .}} +{{- else if .Values.thanosRuler.thanosRulerSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }} + alertQueryUrl: "{{ .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }}" +{{- end}} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl }} + alertmanagersUrl: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret }} + alertmanagersConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.secret }} + alertmanagersConfig: + key: alertmanager-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryEndpoints }} + queryEndpoints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.queryEndpoints | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret }} + queryConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.queryConfig.secret }} + queryConfig: + key: query-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.resources }} + resources: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.routePrefix }} + routePrefix: "{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.securityContext }} + securityContext: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.storage }} + storage: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.storage | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret }} + objectStorageConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.secret }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.labels }} + labels: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.thanosRuler.thanosRulerSpec.podAntiAffinity .Values.thanosRuler.thanosRulerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.affinity }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- else if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.tolerations }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.containers }} + containers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.initContainers }} + initContainers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.priorityClassName }} + priorityClassName: {{.Values.thanosRuler.thanosRulerSpec.priorityClassName }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumes }} + volumes: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.thanosRuler.thanosRulerSpec.portName }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/secret.yaml new file mode 100644 index 0000000000..acab7fd9ae --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/secret.yaml @@ -0,0 +1,26 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + {{- with .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig }} + {{- if and .secret (not .existingSecret) }} + alertmanager-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.objectStorageConfig }} + {{- if and .secret (not .existingSecret) }} + object-storage-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.queryConfig }} + {{- if and .secret (not .existingSecret) }} + query-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/service.yaml new file mode 100644 index 0000000000..be0c844591 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/service.yaml @@ -0,0 +1,53 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.service.labels }} +{{ toYaml .Values.thanosRuler.service.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.service.clusterIP }} + clusterIP: {{ .Values.thanosRuler.service.clusterIP }} +{{- end }} +{{- if .Values.thanosRuler.service.externalIPs }} + externalIPs: +{{ toYaml .Values.thanosRuler.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.thanosRuler.service.loadBalancerIP }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.thanosRuler.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.thanosRuler.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.thanosRuler.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if eq .Values.thanosRuler.service.type "NodePort" }} + nodePort: {{ .Values.thanosRuler.service.nodePort }} + {{- end }} + port: {{ .Values.thanosRuler.service.port }} + targetPort: {{ .Values.thanosRuler.service.targetPort }} + protocol: TCP +{{- if .Values.thanosRuler.service.additionalPorts }} +{{ toYaml .Values.thanosRuler.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + type: "{{ .Values.thanosRuler.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/serviceaccount.yaml new file mode 100644 index 0000000000..b58f1cd4df --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/servicemonitor.yaml new file mode 100644 index 0000000000..b2b138b498 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/servicemonitor.yaml @@ -0,0 +1,82 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.thanosRuler.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.thanosRuler.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + release: {{ $.Release.Name | quote }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if .Values.thanosRuler.serviceMonitor.interval }} + interval: {{ .Values.thanosRuler.serviceMonitor.interval }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.thanosRuler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.scheme }} + scheme: {{ .Values.thanosRuler.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.thanosRuler.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.thanosRuler.thanosRulerSpec.routePrefix }}/metrics" + {{- if .Values.thanosRuler.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.thanosRuler.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.thanosRuler.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.thanosRuler.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.thanosRuler.serviceMonitor.interval .interval }} + interval: {{ default $.Values.thanosRuler.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.thanosRuler.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..6fcb8b3a69 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-install-crd.yaml @@ -0,0 +1,23 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/AlertmanagerConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/Alertmanager" false -}} +# {{- set $found "monitoring.coreos.com/v1/PodMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/Probe" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/PrometheusAgent" false -}} +# {{- set $found "monitoring.coreos.com/v1/Prometheus" false -}} +# {{- set $found "monitoring.coreos.com/v1/PrometheusRule" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/ScrapeConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/ThanosRuler" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/values.yaml new file mode 100644 index 0000000000..0bf19a8572 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/values.yaml @@ -0,0 +1,5431 @@ +# Default values for kube-prometheus-stack. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Rancher Monitoring Configuration + +## Configuration for prometheus-adapter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter +## +prometheus-adapter: + enabled: true + prometheus: + # Change this if you change the namespaceOverride or nameOverride of prometheus-operator + url: http://rancher-monitoring-prometheus.cattle-monitoring-system.svc + port: 9090 + +## RKE PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rkeControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.23" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.23 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeEtcd: + enabled: false + metricsPort: 2379 + component: kube-etcd + clients: + port: 10014 + https: + enabled: true + certDir: /etc/kubernetes/ssl + certFile: kube-etcd-*.pem + keyFile: kube-etcd-*-key.pem + caCertFile: kube-ca.pem + seLinuxOptions: + # Gives rkeEtcd permissions to read files in /etc/kubernetes/* + # Type is defined in https://github.com/rancher/rancher-selinux + type: rke_kubereader_t + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeIngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + clients: + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + nodeSelector: + node-role.kubernetes.io/worker: "true" + +## k3s PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +k3sServer: + enabled: false + metricsPort: 10250 + component: k3s-server + clients: + port: 10013 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +hardened: + k3s: + networkPolicy: + enabled: true + +## KubeADM PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +kubeAdmControllerManager: + enabled: false + metricsPort: 10257 + component: kube-controller-manager + clients: + port: 10011 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + port: 10012 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmEtcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## rke2 PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rke2ControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Scheduler: + enabled: false + metricsPort: 10259 # default to secure port as of k8s >= 1.22 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Proxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2Etcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2IngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + # in the RKE2 cluster, the ingress-nginx-controller is deployed + # as a non-hostNetwork workload starting at the following versions + # - >= v1.22.12+rke2r1 < 1.23.0-0 + # - >= v1.23.9+rke2r1 < 1.24.0-0 + # - >= v1.24.3+rke2r1 < 1.25.0-0 + # - >= v1.25.0+rke2r1 + # As a result we do not need clients and proxies as we can directly create + # a service that targets the workload with the given app name + namespaceOverride: kube-system + clients: + enabled: false + proxy: + enabled: false + service: + selector: + app.kubernetes.io/name: rke2-ingress-nginx + kubeVersionOverrides: + - constraint: "< 1.21.0-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a DaemonSet with 1 pod when RKE2 version is < 1.21.0-0 + deployment: + enabled: false + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.21.0-0 < 1.22.12-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.21.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.23.0-0 < v1.23.9-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.24.0-0 < v1.24.3-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + + + +## Additional PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## + +# hardenedKubelet can only be deployed if kubelet.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default kubelet option with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedKubelet: + enabled: false + metricsPort: 10250 + component: kubelet + clients: + port: 10015 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +# hardenedNodeExporter can only be deployed if nodeExporter.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default nodeExporter with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedNodeExporter: + enabled: false + metricsPort: 9796 + component: node-exporter + clients: + port: 10016 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## Upgrades +upgrade: + ## Run upgrade scripts before an upgrade or rollback via a Job hook + enabled: true + ## Image to use to run the scripts + image: + repository: rancher/shell + tag: v0.2.1 + +## Rancher Monitoring +## + +rancherMonitoring: + enabled: true + + ## A namespaceSelector to identify the namespace to find the Rancher deployment + ## + namespaceSelector: + matchNames: + - cattle-system + + ## A selector to identify the Rancher deployment + ## If not set, the chart will try to search for the Rancher deployment in the cattle-system namespace and infer the selector values from it + ## If the Rancher deployment does not exist, no resources will be deployed. + ## + selector: {} + +## Component scraping nginx-ingress-controller +## +ingressNginx: + enabled: false + + ## The namespace to search for your nginx-ingress-controller + ## + namespace: ingress-nginx + + service: + port: 9913 + targetPort: 10254 + # selector: + # app: ingress-nginx + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "30s" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +# Prometheus Operator Configuration + +## Provide a name in place of kube-prometheus-stack for `app:` labels +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +nameOverride: "rancher-monitoring" + +## Override the deployment namespace +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +namespaceOverride: "cattle-monitoring-system" + +## Provide a k8s version to auto dashboard import script example: kubeTargetVersionOverride: 1.26.6 +## +kubeTargetVersionOverride: "" + +## Allow kubeVersion to be overridden while creating the ingress +## +kubeVersionOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +## Install Prometheus Operator CRDs +## +crds: + enabled: true + +## custom Rules to override "for" and "severity" in defaultRules +## +customRules: {} + # AlertmanagerFailedReload: + # for: 3m + # AlertmanagerMembersInconsistent: + # for: 5m + # severity: "warning" + +## Create default rules for monitoring the cluster +## +defaultRules: + create: true + rules: + alertmanager: true + etcd: true + configReloaders: true + general: true + k8sContainerCpuUsageSecondsTotal: true + k8sContainerMemoryCache: true + k8sContainerMemoryRss: true + k8sContainerMemorySwap: true + k8sContainerResource: true + k8sContainerMemoryWorkingSetBytes: true + k8sPodOwner: true + kubeApiserverAvailability: true + kubeApiserverBurnrate: true + kubeApiserverHistogram: true + kubeApiserverSlos: true + kubeControllerManager: true + kubelet: true + kubeProxy: true + kubePrometheusGeneral: true + kubePrometheusNodeRecording: true + kubernetesApps: true + kubernetesResources: true + kubernetesStorage: true + kubernetesSystem: true + kubeSchedulerAlerting: true + kubeSchedulerRecording: true + kubeStateMetrics: true + network: true + node: true + nodeExporterAlerting: true + nodeExporterRecording: true + prometheus: true + prometheusOperator: true + windows: true + + ## Reduce app namespace alert scope + appNamespacesTarget: ".*" + + ## Set keep_firing_for for all alerts + keepFiringFor: "" + + ## Labels for default rules + labels: {} + ## Annotations for default rules + annotations: {} + + ## Additional labels for PrometheusRule alerts + additionalRuleLabels: {} + + ## Additional annotations for PrometheusRule alerts + additionalRuleAnnotations: {} + + ## Additional labels for specific PrometheusRule alert groups + additionalRuleGroupLabels: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + ## Additional annotations for specific PrometheusRule alerts groups + additionalRuleGroupAnnotations: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + additionalAggregationLabels: [] + + ## Prefix for runbook URLs. Use this to override the first part of the runbookURLs that is common to all rules. + runbookUrl: "https://runbooks.prometheus-operator.dev/runbooks" + + ## Disabled PrometheusRule alerts + disabled: {} + # KubeAPIDown: true + # NodeRAIDDegraded: true + +## Deprecated way to provide custom recording or alerting rules to be deployed into the cluster. +## +# additionalPrometheusRules: [] +# - name: my-rule-file +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## Provide custom recording or alerting rules to be deployed into the cluster. +## +additionalPrometheusRulesMap: {} +# rule-name: +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## +global: + cattle: + psp: + enabled: false + + systemDefaultRegistry: "" + ## Windows Monitoring + ## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-windows-exporter + ## + ## Deploys a DaemonSet of Prometheus exporters based on https://github.com/prometheus-community/windows_exporter. + ## Every Windows host must have a wins version of 0.1.0+ to use this chart (default as of Rancher 2.5.8). + ## To upgrade wins versions on Windows hosts, see https://github.com/rancher/wins/tree/master/charts/rancher-wins-upgrader. + ## + windows: + enabled: false + seLinux: + enabled: false + kubectl: + repository: rancher/kubectl + tag: v1.20.2 + pullPolicy: IfNotPresent + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Global image registry to use if it needs to be overriden for some specific use cases (e.g local registries, custom images, ...) + ## + imageRegistry: docker.io + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + # or + # - "image-pull-secret" + +windowsMonitoring: + ## Deploys the windows-exporter and Windows-specific dashboards and rules (job name must be 'windows-exporter') + enabled: false + +## Configuration for prometheus-windows-exporter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-windows-exporter +## +prometheus-windows-exporter: + ## Enable ServiceMonitor and set Kubernetes label to use as a job label + ## + prometheus: + monitor: + enabled: true + jobLabel: jobLabel + + releaseLabel: true + + ## Set job label to 'windows-exporter' as required by the default Prometheus rules and Grafana dashboards + ## + podLabels: + jobLabel: windows-exporter + + ## Enable memory and container metrics as required by the default Prometheus rules and Grafana dashboards + ## + config: |- + collectors: + enabled: '[defaults],memory,container' + +## Configuration for alertmanager +## ref: https://prometheus.io/docs/alerting/alertmanager/ +## +alertmanager: + + ## Deploy alertmanager + ## + enabled: true + + ## Annotations for Alertmanager + ## + annotations: {} + + ## Api that prometheus will use to communicate with alertmanager. Possible values are v1, v2 + ## + apiVersion: v2 + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + ## Configure pod disruption budgets for Alertmanager + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + ## Alertmanager configuration directives + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + config: + global: + resolve_timeout: 5m + inhibit_rules: + - source_matchers: + - 'severity = critical' + target_matchers: + - 'severity =~ warning|info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'severity = warning' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'alertname = InfoInhibitor' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - target_matchers: + - 'alertname = InfoInhibitor' + route: + group_by: ['namespace'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'null' + routes: + - receiver: 'null' + matchers: + - alertname = "Watchdog" + receivers: + - name: 'null' + templates: + - '/etc/alertmanager/config/*.tmpl' + + ## Alertmanager configuration directives (as string type, preferred over the config hash map) + ## stringConfig will be used only, if tplConfig is true + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + stringConfig: "" + + ## Pass the Alertmanager configuration directives through Helm's templating + ## engine. If the Alertmanager configuration contains Alertmanager templates, + ## they'll need to be properly escaped so that they are not interpreted by + ## Helm + ## ref: https://helm.sh/docs/developing_charts/#using-the-tpl-function + ## https://prometheus.io/docs/alerting/configuration/#tmpl_string + ## https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + tplConfig: false + + ## Alertmanager template files to format alerts + ## By default, templateFiles are placed in /etc/alertmanager/config/ and if + ## they have a .tmpl file suffix will be loaded. See config.templates above + ## to change, add other suffixes. If adding other suffixes, be sure to update + ## config.templates above to include those suffixes. + ## ref: https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + ## + + templateFiles: + rancher_defaults.tmpl: |- + {{- define "slack.rancher.text" -}} + {{ template "rancher.text_multiple" . }} + {{- end -}} + + {{- define "rancher.text_multiple" -}} + *[GROUP - Details]* + One or more alarms in this group have triggered a notification. + + {{- if gt (len .GroupLabels.Values) 0 }} + *Group Labels:* + {{- range .GroupLabels.SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- if .ExternalURL }} + *Link to AlertManager:* {{ .ExternalURL }} + {{- end }} + + {{- range .Alerts }} + {{ template "rancher.text_single" . }} + {{- end }} + {{- end -}} + + {{- define "rancher.text_single" -}} + {{- if .Labels.alertname }} + *[ALERT - {{ .Labels.alertname }}]* + {{- else }} + *[ALERT]* + {{- end }} + {{- if .Labels.severity }} + *Severity:* `{{ .Labels.severity }}` + {{- end }} + {{- if .Labels.cluster }} + *Cluster:* {{ .Labels.cluster }} + {{- end }} + {{- if .Annotations.summary }} + *Summary:* {{ .Annotations.summary }} + {{- end }} + {{- if .Annotations.message }} + *Message:* {{ .Annotations.message }} + {{- end }} + {{- if .Annotations.description }} + *Description:* {{ .Annotations.description }} + {{- end }} + {{- if .Annotations.runbook_url }} + *Runbook URL:* <{{ .Annotations.runbook_url }}|:spiral_note_pad:> + {{- end }} + {{- with .Labels }} + {{- with .Remove (stringSlice "alertname" "severity" "cluster") }} + {{- if gt (len .) 0 }} + *Additional Labels:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Annotations }} + {{- with .Remove (stringSlice "summary" "message" "description" "runbook_url") }} + {{- if gt (len .) 0 }} + *Additional Annotations:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end -}} + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + + labels: {} + + ## Override ingress to a different defined port on the service + # servicePort: 8081 + ## Override ingress to a different service then the default, this is useful if you need to + ## point to a specific instance of the alertmanager (eg kube-prometheus-stack-alertmanager-0) + # serviceName: kube-prometheus-stack-alertmanager-0 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - alertmanager.domain.com + + ## Paths to use for ingress rules - one path should match the alertmanagerSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Alertmanager Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: alertmanager-general-tls + # hosts: + # - alertmanager.example.com + + ## Configuration for Alertmanager secret + ## + secret: + annotations: {} + + # by default the alertmanager secret is not overwritten if it already exists + recreateIfExists: false + + ## Configuration for creating an Ingress that will map to each Alertmanager replica service + ## alertmanager.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for alertmanager per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "alertmanager" + + ## Configuration for Alertmanager service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Alertmanager Service to listen on + ## + port: 9093 + ## To be used with a proxy extraContainer port + ## + targetPort: 9093 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30903 + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + + ## Additional ports to open for Alertmanager service + ## + additionalPorts: [] + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a separate Service for each statefulset Alertmanager replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Alertmanager Service per replica to listen on + ## + port: 9093 + + ## To be used with a proxy extraContainer port + targetPort: 9093 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30904 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a ServiceMonitor for AlertManager + ## + serviceMonitor: + ## If true, a ServiceMonitor will be created for the AlertManager service. + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## enableHttp2: Whether to enable HTTP2. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint + enableHttp2: true + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting alertmanagerSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec + ## + alertmanagerSpec: + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the Alertmanager pods. + ## + podMetadata: {} + + ## Image of Alertmanager + ## + image: + repository: rancher/mirrored-prometheus-alertmanager + tag: v0.27.0 + sha: "" + + ## If true then the user will be responsible to provide a secret with alertmanager configuration + ## So when true the config part will be ignored (including templateFiles) and the one in the secret will be used + ## + useExistingSecret: false + + ## Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the + ## Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/. + ## + secrets: [] + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. + ## The ConfigMaps are mounted into /etc/alertmanager/configmaps/. + ## + configMaps: [] + + ## ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for + ## this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config. + ## + # configSecret: + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerwebspec + web: {} + + ## AlertmanagerConfigs to be selected to merge and configure Alertmanager with. + ## + alertmanagerConfigSelector: {} + ## Example which selects all alertmanagerConfig resources + ## with label "alertconfig" with values any of "example-config" or "example-config-2" + # alertmanagerConfigSelector: + # matchExpressions: + # - key: alertconfig + # operator: In + # values: + # - example-config + # - example-config-2 + # + ## Example which selects all alertmanagerConfig resources with label "role" set to "example-config" + # alertmanagerConfigSelector: + # matchLabels: + # role: example-config + + ## Namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace. + ## + alertmanagerConfigNamespaceSelector: {} + ## Example which selects all namespaces + ## with label "alertmanagerconfig" with values any of "example-namespace" or "example-namespace-2" + # alertmanagerConfigNamespaceSelector: + # matchExpressions: + # - key: alertmanagerconfig + # operator: In + # values: + # - example-namespace + # - example-namespace-2 + + ## Example which selects all namespaces with label "alertmanagerconfig" set to "enabled" + # alertmanagerConfigNamespaceSelector: + # matchLabels: + # alertmanagerconfig: enabled + + ## AlermanagerConfig to be used as top level configuration + ## + alertmanagerConfiguration: {} + ## Example with select a global alertmanagerconfig + # alertmanagerConfiguration: + # name: global-alertmanager-Configuration + + ## Defines the strategy used by AlertmanagerConfig objects to match alerts. eg: + ## + alertmanagerConfigMatcherStrategy: {} + ## Example with use OnNamespace strategy + # alertmanagerConfigMatcherStrategy: + # type: OnNamespace + + ## Define Log Format + # Use logfmt (default) or json logging + logFormat: logfmt + + ## Log level for Alertmanager to be configured with. + ## + logLevel: info + + ## Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the + ## running cluster equal to the expected size. + replicas: 1 + + ## Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression + ## [0-9]+(ms|s|m|h) (milliseconds seconds minutes hours). + ## + retention: 120h + + ## Storage is the definition of how storage will be used by the Alertmanager instances. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storage: {} + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + + ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false + ## + externalUrl: + + ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, + ## but the server serves requests under a different route prefix. For example for use with kubectl proxy. + ## + routePrefix: / + + ## scheme: HTTP scheme to use. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when connect to the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + ## If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. + ## + paused: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Define resources requests and limits for single Pods. + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: + memory: 500Mi + cpu: 1000m + requests: + memory: 100Mi + cpu: 100m + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + ## + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the alertmanager instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## If specified, the pod's tolerations. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: alertmanager + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. + ## Note this is only for the Alertmanager UI, not the gossip communication. + ## + listenLocal: false + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. + ## + containers: [] + # containers: + # - name: oauth-proxy + # image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1 + # args: + # - --upstream=http://127.0.0.1:9093 + # - --http-address=0.0.0.0:8081 + # - --metrics-address=0.0.0.0:8082 + # - ... + # ports: + # - containerPort: 8081 + # name: oauth-proxy + # protocol: TCP + # - containerPort: 8082 + # name: oauth-metrics + # protocol: TCP + # resources: {} + + # Additional volumes on the output StatefulSet definition. + volumes: [] + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. + ## + additionalPeers: [] + + ## PortName to use for Alert Manager. + ## + portName: "http-web" + + ## ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 + ## + clusterAdvertiseAddress: false + + ## clusterGossipInterval determines interval between gossip attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterGossipInterval: "" + + ## clusterPeerTimeout determines timeout for cluster peering. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPeerTimeout: "" + + ## clusterPushpullInterval determines interval between pushpull attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPushpullInterval: "" + + ## ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. + ## Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. + forceEnableClusterMode: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use alertmanager.alertmanagerSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + +## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +## +grafana: + enabled: true + namespaceOverride: "" + + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + grafana.ini: + users: + auto_assign_org_role: Viewer + auth: + disable_login_form: false + auth.anonymous: + enabled: true + org_role: Viewer + auth.basic: + enabled: false + dashboards: + # Modify this value to change the default dashboard shown on the main Grafana page + default_home_dashboard_path: /tmp/dashboards/rancher-default-home.json + security: + # Required to embed dashboards in Rancher Cluster Overview Dashboard on Cluster Explorer + allow_embedding: true + + deploymentStrategy: + type: Recreate + + ## ForceDeployDatasources Create datasource configmap even if grafana deployment has been disabled + ## + forceDeployDatasources: false + + ## ForceDeployDashboard Create dashboard configmap even if grafana deployment has been disabled + ## + forceDeployDashboards: false + + ## Deploy default dashboards + ## + defaultDashboardsEnabled: true + + # Additional options for defaultDashboards + defaultDashboards: + # The default namespace to place defaultDashboards within + namespace: cattle-dashboards + # Whether to create the default namespace as a Helm managed namespace or use an existing namespace + # If false, the defaultDashboards.namespace will be created as a Helm managed namespace + useExistingNamespace: false + # Whether the Helm managed namespace created by this chart should be left behind on a Helm uninstall + # If you place other dashboards in this namespace, then they will be deleted on a helm uninstall + # Ignore if useExistingNamespace is true + cleanupOnUninstall: false + + ## Timezone for the default dashboards + ## Other options are: browser or a specific timezone, i.e. Europe/Luxembourg + ## + defaultDashboardsTimezone: utc + + ## Editable flag for the default dashboards + ## + defaultDashboardsEditable: true + + adminPassword: prom-operator + + ingress: + ## If true, Grafana Ingress will be created + ## + enabled: false + + ## IngressClassName for Grafana Ingress. + ## Should be provided if Ingress is enable. + ## + # ingressClassName: nginx + + ## Annotations for Grafana Ingress + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + + ## Labels to be added to the Ingress + ## + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enable. + ## + # hosts: + # - grafana.domain.com + hosts: [] + + ## Path for grafana ingress + path: / + + ## TLS configuration for grafana Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: grafana-general-tls + # hosts: + # - grafana.example.com + + # # To make Grafana persistent (Using Statefulset) + # # + # persistence: + # enabled: true + # type: sts + # storageClassName: "storageClassName" + # accessModes: + # - ReadWriteOnce + # size: 20Gi + # finalizers: + # - kubernetes.io/pvc-protection + + serviceAccount: + create: true + autoMount: true + + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + searchNamespace: cattle-dashboards + labelValue: "1" + + # Support for new table panels, when enabled grafana auto migrates the old table panels to newer table panels + enableNewTablePanelSyntax: false + + ## Annotations for Grafana dashboard configmaps + ## + annotations: {} + multicluster: + global: + enabled: false + etcd: + enabled: false + provider: + allowUiUpdates: false + datasources: + enabled: true + defaultDatasourceEnabled: true + isDefaultDatasource: true + + uid: prometheus + + ## URL of prometheus datasource + ## + # url: http://prometheus-stack-prometheus:9090/ + + ## Prometheus request timeout in seconds + # timeout: 30 + + # If not defined, will use prometheus.prometheusSpec.scrapeInterval or its default + # defaultDatasourceScrapeInterval: 15s + + ## Annotations for Grafana datasource configmaps + ## + annotations: {} + + ## Set method for HTTP to send query to datasource + httpMethod: POST + + ## Create datasource for each Pod of Prometheus StatefulSet; + ## this uses headless service `prometheus-operated` which is + ## created by Prometheus Operator + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/0fee93e12dc7c2ea1218f19ae25ec6b893460590/pkg/prometheus/statefulset.go#L255-L286 + createPrometheusReplicasDatasources: false + label: grafana_datasource + labelValue: "1" + + ## Field with internal link pointing to existing data source in Grafana. + ## Can be provisioned via additionalDataSources + exemplarTraceIdDestinations: {} + # datasourceUid: Jaeger + # traceIdLabelName: trace_id + alertmanager: + enabled: true + uid: alertmanager + handleGrafanaManagedAlerts: false + implementation: prometheus + + extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # configMap: certs-configmap + # readOnly: true + + deleteDatasources: [] + # - name: example-datasource + # orgId: 1 + + ## Configure additional grafana datasources (passed through tpl) + ## ref: http://docs.grafana.org/administration/provisioning/#datasources + additionalDataSources: [] + # - name: prometheus-sample + # access: proxy + # basicAuth: true + # basicAuthPassword: pass + # basicAuthUser: daco + # editable: false + # jsonData: + # tlsSkipVerify: true + # orgId: 1 + # type: prometheus + # url: https://{{ printf "%s-prometheus.svc" .Release.Name }}:9090 + # version: 1 + + ## Passed to grafana subchart and used by servicemonitor below + ## + service: + portName: nginx-http + ## Port for Grafana Service to listen on + ## + port: 80 + ## To be used with a proxy extraContainer port + ## + targetPort: 8080 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30950 + ## Service type + ## + type: ClusterIP + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod + extraContainers: | + - name: grafana-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }}" + ports: + - containerPort: 8080 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: grafana-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## Volumes that can be used in containers + extraContainerVolumes: + - name: nginx-home + emptyDir: {} + - name: grafana-nginx + configMap: + name: grafana-nginx-proxy-config + items: + - key: nginx.conf + mode: 438 + path: nginx.conf + + ## If true, create a serviceMonitor for grafana + ## + serviceMonitor: + # If true, a ServiceMonitor CRD is created for a prometheus operator + # https://github.com/coreos/prometheus-operator + # + enabled: true + + # Path to use for scraping metrics. Might be different if server.root_url is set + # in grafana.ini + path: "/metrics" + + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + + # labels for the ServiceMonitor + labels: {} + + # Scrape interval. If not set, the Prometheus default scrape interval is used. + # + interval: "" + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + resources: + limits: + memory: 200Mi + cpu: 200m + requests: + memory: 100Mi + cpu: 100m + + testFramework: + enabled: false + +## Flag to disable all the kubernetes component scrapers +## +kubernetesServiceMonitors: + enabled: true + +## Component scraping the kube api server +## +kubeApiServer: + enabled: true + tlsConfig: + serverName: kubernetes + insecureSkipVerify: false + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: component + selector: + matchLabels: + component: apiserver + provider: kubernetes + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: + # Drop excessively noisy apiserver buckets. + - action: drop + regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50) + sourceLabels: + - __name__ + - le + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: + # - __meta_kubernetes_namespace + # - __meta_kubernetes_service_name + # - __meta_kubernetes_endpoint_port_name + # action: keep + # regex: default;kubernetes;https + # - targetLabel: __address__ + # replacement: kubernetes.default.svc:443 + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kubelet and kubelet-hosted cAdvisor +## +kubelet: + enabled: true + namespace: kube-system + + serviceMonitor: + ## Attach metadata to discovered targets. Requires Prometheus v2.45 for endpoints created by the operator. + ## + attachMetadata: + node: false + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## If true, Prometheus use (respect) labels provided by exporter. + ## + honorLabels: true + + ## If true, Prometheus ingests metrics with timestamp provided by exporter. If false, Prometheus ingests metrics with timestamp of scrape. + ## + honorTimestamps: true + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Enable scraping the kubelet over https. For requirements to enable this see + ## https://github.com/prometheus-operator/prometheus-operator/issues/926 + ## + https: true + + ## Enable scraping /metrics/cadvisor from kubelet's service + ## + cAdvisor: true + + ## Enable scraping /metrics/probes from kubelet's service + ## + probes: true + + ## Enable scraping /metrics/resource from kubelet's service + ## This is disabled by default because container metrics are already exposed by cAdvisor + ## + resource: false + # From kubernetes 1.18, /metrics/resource/v1alpha1 renamed to /metrics/resource + resourcePath: "/metrics/resource/v1alpha1" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + cAdvisorMetricRelabelings: + # Drop less useful container CPU metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_cpu_(cfs_throttled_seconds_total|load_average_10s|system_seconds_total|user_seconds_total)' + # Drop less useful container / always zero filesystem metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_fs_(io_current|io_time_seconds_total|io_time_weighted_seconds_total|reads_merged_total|sector_reads_total|sector_writes_total|writes_merged_total)' + # Drop less useful / always zero container memory metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_memory_(mapped_file|swap)' + # Drop less useful container process metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_(file_descriptors|tasks_state|threads_max)' + # Drop container spec metrics that overlap with kube-state-metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_spec.*' + # Drop cgroup metrics with no pod. + - sourceLabels: [id, pod] + action: drop + regex: '.+;' + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesMetricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + cAdvisorRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + resourceRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + relabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kube controller manager +## +kubeControllerManager: + enabled: false + + ## If your kube controller manager is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeControllerManager.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.22. + ## + port: null + targetPort: null + # selector: + # component: kube-controller-manager + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-controller-manager + + ## Enable scraping kube-controller-manager over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + # Skip TLS certificate validation when scraping + insecureSkipVerify: null + + # Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping coreDns. Use either this or kubeDns +## +coreDns: + enabled: true + service: + enabled: true + port: 9153 + targetPort: 9153 + # selector: + # k8s-app: kube-dns + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kubeDns. Use either this or coreDns +## +kubeDns: + enabled: false + service: + dnsmasq: + port: 10054 + targetPort: 10054 + skydns: + port: 10055 + targetPort: 10055 + # selector: + # k8s-app: kube-dns + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqMetricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqRelabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping etcd +## +kubeEtcd: + enabled: false + + ## If your etcd is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## Etcd service. If using kubeEtcd.endpoints only the port and targetPort are used + ## + service: + enabled: true + port: 2381 + targetPort: 2381 + # selector: + # component: etcd + + ## Configure secure access to the etcd cluster by loading a secret into prometheus and + ## specifying security configuration below. For example, with a secret named etcd-client-cert + ## + ## serviceMonitor: + ## scheme: https + ## insecureSkipVerify: false + ## serverName: localhost + ## caFile: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + ## certFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client + ## keyFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + ## + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + scheme: http + insecureSkipVerify: false + serverName: "" + caFile: "" + certFile: "" + keyFile: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: etcd + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube scheduler +## +kubeScheduler: + enabled: false + + ## If your kube scheduler is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeScheduler.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.23. + ## + port: null + targetPort: null + # selector: + # component: kube-scheduler + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + ## Enable scraping kube-scheduler over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-scheduler + + ## Skip TLS certificate validation when scraping + insecureSkipVerify: null + + ## Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube proxy +## +kubeProxy: + enabled: false + + ## If your kube proxy is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + service: + enabled: true + port: 10249 + targetPort: 10249 + # selector: + # k8s-app: kube-proxy + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-proxy + + ## Enable scraping kube-proxy over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## + https: false + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube state metrics +## +kubeStateMetrics: + enabled: true + +## Configuration for kube-state-metrics subchart +## +kube-state-metrics: + namespaceOverride: "" + rbac: + create: true + releaseLabel: true + prometheus: + monitor: + enabled: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape Timeout. If not set, the Prometheus default scrape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + # Keep labels from scraped data, overriding server-side labels + ## + honorLabels: true + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + selfMonitor: + enabled: false + +## Deploy node exporter as a daemonset to all nodes +## +nodeExporter: + enabled: true + operatingSystems: + linux: + enabled: true + darwin: + enabled: true + + ## ForceDeployDashboard Create dashboard configmap even if nodeExporter deployment has been disabled + ## + forceDeployDashboards: false + +## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + namespaceOverride: "" + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + releaseLabel: true + extraArgs: + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ + service: + portName: http-metrics + prometheus: + monitor: + enabled: true + + jobLabel: jobLabel + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__] + # separator: ; + # regex: ^node_mountstats_nfs_(event|operations|transport)_.+ + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Manages Prometheus and Alertmanager components +## +prometheusOperator: + enabled: true + + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-operator' by default + fullnameOverride: "" + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Strategy of the deployment + ## + strategy: {} + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # Users who are deploying this chart in GKE private clusters will need to add firewall rules to expose this port for admissions webhooks + internalPort: 8443 + + ## Admission webhook support for PrometheusRules resources added in Prometheus Operator 0.30 can be enabled to prevent incorrectly formatted + ## rules from making their way into prometheus and potentially preventing the container from starting + admissionWebhooks: + ## Valid values: Fail, Ignore, IgnoreOnInstallOnly + ## IgnoreOnInstallOnly - If Release.IsInstall returns "true", set "Ignore" otherwise "Fail" + failurePolicy: "" + ## The default timeoutSeconds is 10 and the maximum value is 30. + timeoutSeconds: 10 + enabled: true + ## A PEM encoded CA bundle which will be used to validate the webhook's server certificate. + ## If unspecified, system trust roots on the apiserver are used. + caBundle: "" + ## If enabled, generate a self-signed certificate, then patch the webhook configurations with the generated data. + ## On chart upgrades (or if the secret exists) the cert will not be re-generated. You can use this to provide your own + ## certs ahead of time if you wish. + ## + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + + namespaceSelector: {} + + deployment: + enabled: false + + ## Number of replicas + ## + replicas: 1 + + ## Strategy of the deployment + ## + strategy: {} + + # Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: {} + # maxUnavailable: 1 + # minAvailable: 1 + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # The default webhook port is 10250 in order to work out-of-the-box in GKE private clusters and avoid adding firewall rules. + internalPort: 10250 + + ## Service account for Prometheus Operator Webhook to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + automountServiceAccountToken: false + create: true + name: "" + + ## Configuration for Prometheus operator Webhook service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 31080 + + nodePortTls: 31443 + + ## Additional ports to open for Prometheus operator Webhook service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator webhook deployment + # ## + labels: {} + + ## Annotations to add to the operator webhook deployment + ## + annotations: {} + + ## Labels to add to the operator webhook pod + ## + podLabels: {} + + ## Annotations to add to the operator webhook pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + ## Prometheus-operator webhook image + ## + image: + registry: quay.io + repository: rancher/mirrored-prometheus-operator-admission-webhook + # if not set appVersion field from Chart.yaml is used + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + + ## Liveness probe + ## + livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Readiness probe + ## + readinessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Resource limits & requests + ## + resources: {} + # limits: + # cpu: 200m + # memory: 200Mi + # requests: + # cpu: 100m + # memory: 100Mi + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + patch: + enabled: true + image: + repository: rancher/mirrored-ingress-nginx-kube-webhook-certgen + tag: v20221220-controller-v1.5.1-58-g787ea74b6 + sha: "" + pullPolicy: IfNotPresent + resources: {} + ## Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + podAnnotations: {} + nodeSelector: {} + affinity: {} + tolerations: [] + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 2000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + seccompProfile: + type: RuntimeDefault + + # Security context for create job container + createSecretJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Security context for patch job container + patchWebhookJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Use certmanager to generate webhook certs + certManager: + enabled: false + # self-signed root certificate + rootCert: + duration: "" # default to be 5y + admissionCert: + duration: "" # default to be 1y + # issuerRef: + # name: "issuer" + # kind: "ClusterIssuer" + + ## Namespaces to scope the interaction of the Prometheus Operator and the apiserver (allow list). + ## This is mutually exclusive with denyNamespaces. Setting this to an empty object will disable the configuration + ## + namespaces: {} + # releaseNamespace: true + # additional: + # - kube-system + + ## Namespaces not to scope the interaction of the Prometheus Operator (deny list). + ## + denyNamespaces: [] + + ## Filter namespaces to look for prometheus-operator custom resources + ## + alertmanagerInstanceNamespaces: [] + alertmanagerConfigNamespaces: [] + prometheusInstanceNamespaces: [] + thanosRulerInstanceNamespaces: [] + + ## The clusterDomain value will be added to the cluster.peer option of the alertmanager. + ## Without this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated:9094 (default value) + ## With this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated.namespace.svc.cluster-domain:9094 + ## + # clusterDomain: "cluster.local" + + networkPolicy: + ## Enable creation of NetworkPolicy resources. + ## + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # egress: + + ## match labels used in selector + # matchLabels: {} + + ## Service account for Prometheus Operator to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + + ## Configuration for Prometheus operator service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30080 + + nodePortTls: 30443 + + ## Additional ports to open for Prometheus operator service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator deployment + # ## + labels: {} + + ## Annotations to add to the operator deployment + ## + annotations: {} + + ## Labels to add to the operator pod + ## + podLabels: {} + + ## Annotations to add to the operator pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + kubeletService: + ## If true, the operator will create and maintain a service for scraping kubelets + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/helm/prometheus-operator/README.md + ## + enabled: true + namespace: kube-system + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-kubelet' by default + name: "" + + ## Create a servicemonitor for the operator + ## + serviceMonitor: + ## If true, create a serviceMonitor for prometheus operator + ## + selfMonitor: true + + ## Labels for ServiceMonitor + additionalLabels: {} + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape timeout. If not set, the Prometheus default scrape timeout is used. + scrapeTimeout: "" + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Resource limits & requests + ## + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + + ## Operator Environment + ## env: + ## VARIABLE: value + env: + GOGC: "30" + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Enable vertical pod autoscaler support for prometheus-operator + verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + updateMode: Auto + + ## Prometheus-operator image + ## + image: + repository: rancher/mirrored-prometheus-operator-prometheus-operator + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Prometheus image to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImage: prometheus/prometheus + + ## Prometheus image registry to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImageRegistry: quay.io + + ## Alertmanager image to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImage: prometheus/alertmanager + + ## Alertmanager image registry to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImageRegistry: quay.io + + ## Prometheus-config-reloader + ## + prometheusConfigReloader: + image: + repository: rancher/mirrored-prometheus-operator-prometheus-config-reloader + tag: v0.72.0 + sha: "" + + # add prometheus config reloader liveness and readiness probe. Default: false + enableProbe: false + + # resource config for prometheusConfigReloader + resources: {} + # requests: + # cpu: 200m + # memory: 50Mi + # limits: + # cpu: 200m + # memory: 50Mi + + ## Thanos side-car image when configured + ## + thanosImage: + repository: rancher/mirrored-thanos-thanos + tag: v0.34.1 + sha: "" + + ## Set a Label Selector to filter watched prometheus and prometheusAgent + ## + prometheusInstanceSelector: "" + + ## Set a Label Selector to filter watched alertmanager + ## + alertmanagerInstanceSelector: "" + + ## Set a Label Selector to filter watched thanosRuler + thanosRulerInstanceSelector: "" + + ## Set a Field Selector to filter watched secrets + ## + secretFieldSelector: "type!=kubernetes.io/dockercfg,type!=kubernetes.io/service-account-token,type!=helm.sh/release.v1" + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## Additional volumes + ## + extraVolumes: [] + + ## Additional volume mounts + ## + extraVolumeMounts: [] + +## Deploy a Prometheus instance +## +prometheus: + enabled: true + + ## Toggle prometheus into agent mode + ## Note many of features described below (e.g. rules, query, alerting, remote read, thanos) will not work in agent mode. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/designs/prometheus-agent.md + ## + agentMode: false + + ## Annotations for Prometheus + ## + annotations: {} + + ## Configure network policy for the prometheus + networkPolicy: + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # endpointSelector: + # egress: + # ingress: + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app: prometheus + + ## Service account for Prometheuses to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + # Service for thanos service discovery on sidecar + # Enable this can make Thanos Query can use + # `--store=dnssrv+_grpc._tcp.${kube-prometheus-stack.fullname}-thanos-discovery.${namespace}.svc.cluster.local` to discovery + # Thanos sidecar on prometheus nodes + # (Please remember to change ${kube-prometheus-stack.fullname} and ${namespace}. Not just copy and paste!) + thanosService: + enabled: false + annotations: {} + labels: {} + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## ClusterIP to assign + # Default is to make this a headless service ("None") + clusterIP: "None" + + ## Port to expose on each node, if service type is NodePort + ## + nodePort: 30901 + httpNodePort: 30902 + + # ServiceMonitor to scrape Sidecar metrics + # Needs thanosService to be enabled as well + thanosServiceMonitor: + enabled: false + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + metricRelabelings: [] + + ## relabel configs to apply to samples before ingestion. + relabelings: [] + + # Service for external access to sidecar + # Enabling this creates a service to expose thanos-sidecar outside the cluster. + thanosServiceExternal: + enabled: false + annotations: {} + labels: {} + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: LoadBalancer + + ## Port to expose on each node + ## + nodePort: 30901 + httpNodePort: 30902 + + ## Configuration for Prometheus service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Prometheus Service to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 8081 + + ## Port for Prometheus Reloader to listen on + ## + reloaderWebPort: 8080 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30090 + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Additional ports to open for Prometheus service + ## + additionalPorts: [] + # additionalPorts: + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + ## Consider that all endpoints are considered "ready" even if the Pods themselves are not + ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec + publishNotReadyAddresses: false + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Configuration for creating a separate Service for each statefulset Prometheus replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Prometheus Service per replica to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30091 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configure pod disruption budgets for Prometheus + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + # Ingress exposes thanos sidecar outside the cluster + thanosIngress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + servicePort: 10901 + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30901 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - thanos-gateway.domain.com + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Thanos Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: thanos-gateway-tls + # hosts: + # - thanos-gateway.domain.com + # + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Redirect ingress to an additional defined port on the service + # servicePort: 8081 + + ## Hostnames. + ## Must be provided if Ingress is enabled. + ## + # hosts: + # - prometheus.domain.com + hosts: [] + + ## Paths to use for ingress rules - one path should match the prometheusSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Prometheus Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: prometheus-general-tls + # hosts: + # - prometheus.example.com + + ## Configuration for creating an Ingress that will map to each Prometheus replica service + ## prometheus.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for Prometheus per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "prometheus" + + ## Configure additional options for default pod security policy for Prometheus + ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + allowedCapabilities: [] + allowedHostPaths: [] + volumes: [] + + serviceMonitor: + ## If true, create a serviceMonitor for prometheus + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting prometheusSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec + ## + prometheusSpec: + ## If true, pass --storage.tsdb.max-block-duration=2h to prometheus. This is already done if using Thanos + ## + disableCompaction: false + ## APIServerConfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#apiserverconfig + ## + apiserverConfig: {} + + ## Allows setting additional arguments for the Prometheus container + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.Prometheus + additionalArgs: [] + + ## Interval between consecutive scrapes. + ## Defaults to 30s. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/release-0.44/pkg/prometheus/promcfg.go#L180-L183 + ## + scrapeInterval: "30s" + + ## Number of seconds to wait for target to respond before erroring + ## + # scrapeTimeout: "30s" + + ## Interval between consecutive evaluations. + ## + evaluationInterval: "30s" + + ## ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. + ## + listenLocal: false + + ## EnableAdminAPI enables Prometheus the administrative HTTP API which includes functionality such as deleting time series. + ## This is disabled by default. + ## ref: https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis + ## + enableAdminAPI: false + + ## Sets version of Prometheus overriding the Prometheus version as derived + ## from the image tag. Useful in cases where the tag does not follow semver v2. + version: "" + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#webtlsconfig + web: {} + + ## Exemplars related settings that are runtime reloadable. + ## It requires to enable the exemplar storage feature to be effective. + exemplars: "" + ## Maximum number of exemplars stored in memory for all series. + ## If not set, Prometheus uses its default value. + ## A value of zero or less than zero disables the storage. + # maxSize: 100000 + + # EnableFeatures API enables access to Prometheus disabled features. + # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/ + enableFeatures: [] + # - exemplar-storage + + ## Image of Prometheus. + ## + image: + repository: rancher/mirrored-prometheus-prometheus + tag: v2.50.1 + sha: "" + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: prometheus + + ## Alertmanagers to which alerts will be sent + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerendpoints + ## + ## Default configuration will connect to the alertmanager deployed as part of this release + ## + alertingEndpoints: [] + # - name: "" + # namespace: "" + # port: http + # scheme: http + # pathPrefix: "" + # tlsConfig: {} + # bearerTokenFile: "" + # apiVersion: v2 + + ## External labels to add to any time series or alerts when communicating with external systems + ## + externalLabels: {} + + ## enable --web.enable-remote-write-receiver flag on prometheus-server + ## + enableRemoteWriteReceiver: false + + ## Name of the external label used to denote replica name + ## + replicaExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote replica name + ## + replicaExternalLabelNameClear: false + + ## Name of the external label used to denote Prometheus instance name + ## + prometheusExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote Prometheus instance name + ## + prometheusExternalLabelNameClear: false + + ## External URL at which Prometheus will be reachable. + ## + externalUrl: "" + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not + ## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated + ## with the new list of secrets. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The ConfigMaps are mounted into /etc/prometheus/configmaps/. + ## + configMaps: [] + + ## QuerySpec defines the query command line flags when starting Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#queryspec + ## + query: {} + + ## If nil, select own namespace. Namespaces to be selected for PrometheusRules discovery. + ruleNamespaceSelector: {} + ## Example which selects PrometheusRules in namespaces with label "prometheus" set to "somelabel" + # ruleNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.ruleSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the PrometheusRule resources created + ## + ruleSelectorNilUsesHelmValues: false + + ## PrometheusRules to be selected for target discovery. + ## If {}, select all PrometheusRules + ## + ruleSelector: {} + ## Example which select all PrometheusRules resources + ## with label "prometheus" with values any of "example-rules" or "example-rules-2" + # ruleSelector: + # matchExpressions: + # - key: prometheus + # operator: In + # values: + # - example-rules + # - example-rules-2 + # + ## Example which select all PrometheusRules resources with label "role" set to "example-rules" + # ruleSelector: + # matchLabels: + # role: example-rules + + ## If true, a nil or {} value for prometheus.prometheusSpec.serviceMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the servicemonitors created + ## + serviceMonitorSelectorNilUsesHelmValues: false + + ## ServiceMonitors to be selected for target discovery. + ## If {}, select all ServiceMonitors + ## + serviceMonitorSelector: {} + ## Example which selects ServiceMonitors with label "prometheus" set to "somelabel" + # serviceMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for ServiceMonitor discovery. + ## + serviceMonitorNamespaceSelector: {} + ## Example which selects ServiceMonitors in namespaces with label "prometheus" set to "somelabel" + # serviceMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.podMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the podmonitors created + ## + podMonitorSelectorNilUsesHelmValues: false + + ## PodMonitors to be selected for target discovery. + ## If {}, select all PodMonitors + ## + podMonitorSelector: {} + ## Example which selects PodMonitors with label "prometheus" set to "somelabel" + # podMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for PodMonitor discovery. + podMonitorNamespaceSelector: {} + ## Example which selects PodMonitor in namespaces with label "prometheus" set to "somelabel" + # podMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.probeSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the probes created + ## + probeSelectorNilUsesHelmValues: true + + ## Probes to be selected for target discovery. + ## If {}, select all Probes + ## + probeSelector: {} + ## Example which selects Probes with label "prometheus" set to "somelabel" + # probeSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for Probe discovery. + probeNamespaceSelector: {} + ## Example which selects Probe in namespaces with label "prometheus" set to "somelabel" + # probeNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.scrapeConfigSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the scrapeConfigs created + ## + scrapeConfigSelectorNilUsesHelmValues: true + + ## scrapeConfigs to be selected for target discovery. + ## If {}, select all scrapeConfigs + ## + scrapeConfigSelector: {} + ## Example which selects scrapeConfigs with label "prometheus" set to "somelabel" + # scrapeConfigSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for scrapeConfig discovery. + scrapeConfigNamespaceSelector: {} + ## Example which selects scrapeConfig in namespaces with label "prometheus" set to "somelabel" + # scrapeConfigNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## How long to retain metrics + ## + retention: 10d + + ## Maximum size of metrics + ## + retentionSize: "" + + ## Allow out-of-order/out-of-bounds samples ingested into Prometheus for a specified duration + ## See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb + tsdb: + outOfOrderTimeWindow: 0s + + ## Enable compression of the write-ahead log using Snappy. + ## + walCompression: true + + ## If true, the Operator won't process any Prometheus configuration changes + ## + paused: false + + ## Number of replicas of each shard to deploy for a Prometheus deployment. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## + replicas: 1 + + ## EXPERIMENTAL: Number of shards to distribute targets onto. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. + ## Increasing shards will not reshard data either but it will continue to be available from the same instances. + ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. + ## Sharding is done on the content of the `__address__` target meta-label. + ## + shards: 1 + + ## Log level for Prometheus be configured in + ## + logLevel: info + + ## Log format for Prometheus be configured in + ## + logFormat: logfmt + + ## Prefix used to register routes, overriding externalUrl route. + ## Useful for proxies that rewrite URLs. + ## + routePrefix: / + + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the prometheus pods. + ## + podMetadata: {} + # labels: + # app: prometheus + # k8s-app: prometheus + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the prometheus instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## The remote_read spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotereadspec + remoteRead: [] + # - url: http://remote1/read + ## additionalRemoteRead is appended to remoteRead + additionalRemoteRead: [] + + ## The remote_write spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotewritespec + remoteWrite: [] + # - url: http://remote1/push + ## additionalRemoteWrite is appended to remoteWrite + additionalRemoteWrite: [] + + ## Enable/Disable Grafana dashboards provisioning for prometheus remote write feature + remoteWriteDashboards: false + + ## Resource limits & requests + ## + resources: + limits: + memory: 3000Mi + cpu: 1000m + requests: + memory: 750Mi + cpu: 750m + + ## Prometheus StorageSpec for persistent data + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storageSpec: {} + ## Using PersistentVolumeClaim + ## + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + ## Using tmpfs volume + ## + # emptyDir: + # medium: Memory + + # Additional volumes on the output StatefulSet definition. + volumes: + - name: nginx-home + emptyDir: {} + - name: prometheus-nginx + configMap: + name: prometheus-nginx-proxy-config + defaultMode: 438 + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations + ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form + ## as specified in the official Prometheus documentation: + ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are + ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility + ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible + ## scrape configs are going to break Prometheus after the upgrade. + ## AdditionalScrapeConfigs can be defined as a list or as a templated string. + ## + ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the + ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes + ## + additionalScrapeConfigs: [] + # - job_name: kube-etcd + # kubernetes_sd_configs: + # - role: node + # scheme: https + # tls_config: + # ca_file: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + # cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client + # key_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + # relabel_configs: + # - action: labelmap + # regex: __meta_kubernetes_node_label_(.+) + # - source_labels: [__address__] + # action: replace + # targetLabel: __address__ + # regex: ([^:;]+):(\d+) + # replacement: ${1}:2379 + # - source_labels: [__meta_kubernetes_node_name] + # action: keep + # regex: .*mst.* + # - source_labels: [__meta_kubernetes_node_name] + # action: replace + # targetLabel: node + # regex: (.*) + # replacement: ${1} + # metric_relabel_configs: + # - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone) + # action: labeldrop + # + ## If scrape config contains a repetitive section, you may want to use a template. + ## In the following example, you can see how to define `gce_sd_configs` for multiple zones + # additionalScrapeConfigs: | + # - job_name: "node-exporter" + # gce_sd_configs: + # {{range $zone := .Values.gcp_zones}} + # - project: "project1" + # zone: "{{$zone}}" + # port: 9100 + # {{end}} + # relabel_configs: + # ... + + + ## If additional scrape configurations are already deployed in a single secret file you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalScrapeConfigs + additionalScrapeConfigsSecret: {} + # enabled: false + # name: + # key: + + ## additionalPrometheusSecretsAnnotations allows to add annotations to the kubernetes secret. This can be useful + ## when deploying via spinnaker to disable versioning on the secret, strategy.spinnaker.io/versioned: 'false' + additionalPrometheusSecretsAnnotations: {} + + ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified + ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#. + ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. + ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this + ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release + ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. + ## + additionalAlertManagerConfigs: [] + # - consul_sd_configs: + # - server: consul.dev.test:8500 + # scheme: http + # datacenter: dev + # tag_separator: ',' + # services: + # - metrics-prometheus-alertmanager + + ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertManagerConfigs + additionalAlertManagerConfigsSecret: {} + # name: + # key: + # optional: false + + ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended + ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the + ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. + ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the + ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel + ## configs are going to break Prometheus after the upgrade. + ## + additionalAlertRelabelConfigs: [] + # - separator: ; + # regex: prometheus_replica + # replacement: $1 + # action: labeldrop + + ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertRelabelConfigs + additionalAlertRelabelConfigsSecret: {} + # name: + # key: + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. + ## https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. + ## This section is experimental, it may change significantly without deprecation notice in any release. + ## This is experimental and may change significantly without backward compatibility in any release. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#thanosspec + ## + thanos: {} + # secretProviderClass: + # provider: gcp + # parameters: + # secrets: | + # - resourceName: "projects/$PROJECT_ID/secrets/testsecret/versions/latest" + # fileName: "objstore.yaml" + ## ObjectStorageConfig configures object storage in Thanos. + # objectStorageConfig: + # # use existing secret, if configured, objectStorageConfig.secret will not be used + # existingSecret: {} + # # name: "" + # # key: "" + # # will render objectStorageConfig secret data and configure it to be used by Thanos custom resource, + # # ignored when prometheusspec.thanos.objectStorageConfig.existingSecret is set + # # https://thanos.io/tip/thanos/storage.md/#s3 + # secret: {} + # # type: S3 + # # config: + # # bucket: "" + # # endpoint: "" + # # region: "" + # # access_key: "" + # # secret_key: "" + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. + ## if using proxy extraContainer update targetPort with proxy container port + containers: | + - name: prometheus-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.proxy.image.repository }}:{{ .Values.prometheus.prometheusSpec.proxy.image.tag }}" + ports: + - containerPort: 8081 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: prometheus-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## PortName to use for Prometheus. + ## + portName: "http-web" + + ## ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files + ## on the file system of the Prometheus container e.g. bearer token files. + arbitraryFSAccessThroughSMs: false + + ## OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor + ## or PodMonitor to true, this overrides honor_labels to false. + overrideHonorLabels: false + + ## OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs. + overrideHonorTimestamps: false + + ## When ignoreNamespaceSelectors is set to true, namespaceSelector from all PodMonitor, ServiceMonitor and Probe objects will be ignored, + ## they will only discover targets within the namespace of the PodMonitor, ServiceMonitor and Probe object, + ## and servicemonitors will be installed in the default service namespace. + ## Defaults to false. + ignoreNamespaceSelectors: true + + ## EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. + ## The label value will always be the namespace of the object that is being created. + ## Disabled by default + enforcedNamespaceLabel: "" + + ## PrometheusRulesExcludedFromEnforce - list of prometheus rules to be excluded from enforcing of adding namespace labels. + ## Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair + ## Deprecated, use `excludedFromEnforcement` instead + prometheusRulesExcludedFromEnforce: [] + + ## ExcludedFromEnforcement - list of object references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + ## to be excluded from enforcing a namespace label of origin. + ## Works only if enforcedNamespaceLabel set to true. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#objectreference + excludedFromEnforcement: [] + + ## QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, + ## and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such + ## as /dev/stdout to log querie information to the default Prometheus log stream. This is only available in versions + ## of Prometheus >= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/) + queryLogFile: false + + # Use to set global sample_limit for Prometheus. This act as default SampleLimit for ServiceMonitor or/and PodMonitor. + # Set to 'false' to disable global sample_limit. or set to a number to override the default value. + sampleLimit: false + + # EnforcedKeepDroppedTargetsLimit defines on the number of targets dropped by relabeling that will be kept in memory. + # The value overrides any spec.keepDroppedTargets set by ServiceMonitor, PodMonitor, Probe objects unless spec.keepDroppedTargets + # is greater than zero and less than spec.enforcedKeepDroppedTargets. 0 means no limit. + enforcedKeepDroppedTargets: 0 + + ## EnforcedSampleLimit defines global limit on number of scraped samples that will be accepted. This overrides any SampleLimit + ## set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the SampleLimit to keep overall + ## number of samples/series under the desired limit. Note that if SampleLimit is lower that value will be taken instead. + enforcedSampleLimit: false + + ## EnforcedTargetLimit defines a global limit on the number of scraped targets. This overrides any TargetLimit set + ## per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the TargetLimit to keep the overall + ## number of targets under the desired limit. Note that if TargetLimit is lower, that value will be taken instead, except + ## if either value is zero, in which case the non-zero value will be used. If both values are zero, no limit is enforced. + enforcedTargetLimit: false + + + ## Per-scrape limit on number of labels that will be accepted for a sample. If more than this number of labels are present + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelLimit: false + + ## Per-scrape limit on length of labels name that will be accepted for a sample. If a label name is longer than this number + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelNameLengthLimit: false + + ## Per-scrape limit on length of labels value that will be accepted for a sample. If a label value is longer than this + ## number post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus + ## versions 2.27.0 and newer. + enforcedLabelValueLengthLimit: false + + ## AllowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. This is still experimental + ## in Prometheus so it may change in any upcoming release. + allowOverlappingBlocks: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + # Use the host's network namespace if true. Make sure to understand the security implications if you want to enable it. + # When hostNetwork is enabled, this will set dnsPolicy to ClusterFirstWithHostNet automatically. + hostNetwork: false + + # HostAlias holds the mapping between IP and hostnames that will be injected + # as an entry in the pod’s hosts file. + hostAliases: [] + # - ip: 10.10.0.100 + # hostnames: + # - a1.app.local + # - b1.app.local + + ## TracingConfig configures tracing in Prometheus. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheustracingconfig + tracingConfig: {} + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use prometheus.prometheusSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## Defines the maximum time that the `prometheus` container's startup probe + ## will wait before being considered failed. The startup probe will return + ## success after the WAL replay is complete. If set, the value should be + ## greater than 60 (seconds). Otherwise it will be equal to 600 seconds (15 + ## minutes). + maximumStartupDurationSeconds: 0 + + additionalRulesForClusterRole: [] + # - apiGroups: [ "" ] + # resources: + # - nodes/proxy + # verbs: [ "get", "list", "watch" ] + + additionalServiceMonitors: [] + ## Name of the ServiceMonitor to create + ## + # - name: "" + + ## Additional labels to set used for the ServiceMonitorSelector. Together with standard labels from + ## the chart + ## + # additionalLabels: {} + + ## Service label for use in assembling a job name of the form