From fe212283d75785f67a84e774dc525417c5a85741 Mon Sep 17 00:00:00 2001 From: Eric Promislow Date: Mon, 16 Sep 2024 13:03:35 -0700 Subject: [PATCH] [dev-v2.9] rancher-monitoring - bump chart version --- ...ing-crd-104.1.2-rc.1+up57.0.3.up57.0.3.tgz | Bin 0 -> 299360 bytes ...itoring-104.1.2-rc.1+up57.0.3.up57.0.3.tgz | Bin 0 -> 479983 bytes .../104.1.2-rc.1+up57.0.3.up57.0.3/Chart.yaml | 10 + .../104.1.2-rc.1+up57.0.3.up57.0.3/README.md | 24 + .../files/crd-manifest.tgz | Bin 0 -> 308595 bytes .../templates/_helpers.tpl | 30 + .../templates/jobs.yaml | 102 + .../templates/manifest.yaml | 8 + .../templates/rbac.yaml | 76 + .../templates/validate-psp-install.yaml | 7 + .../values.yaml | 17 + .../.editorconfig | 5 + .../.helmignore | 29 + .../CHANGELOG.md | 47 + .../CONTRIBUTING.md | 12 + .../104.1.2-rc.1+up57.0.3.up57.0.3/Chart.yaml | 126 + .../104.1.2-rc.1+up57.0.3.up57.0.3/README.md | 1080 ++++ .../app-README.md | 46 + .../charts/grafana/.helmignore | 23 + .../charts/grafana/Chart.yaml | 39 + .../charts/grafana/README.md | 770 +++ .../grafana/dashboards/custom-dashboard.json | 1 + .../charts/grafana/templates/NOTES.txt | 55 + .../charts/grafana/templates/_config.tpl | 171 + .../charts/grafana/templates/_helpers.tpl | 305 + .../charts/grafana/templates/_pod.tpl | 1296 ++++ .../charts/grafana/templates/clusterrole.yaml | 25 + .../grafana/templates/clusterrolebinding.yaml | 24 + .../grafana/templates/configSecret.yaml | 43 + .../configmap-dashboard-provider.yaml | 15 + .../charts/grafana/templates/configmap.yaml | 15 + .../templates/dashboards-json-configmap.yaml | 38 + .../charts/grafana/templates/deployment.yaml | 53 + .../grafana/templates/extra-manifests.yaml | 4 + .../grafana/templates/headless-service.yaml | 22 + .../charts/grafana/templates/hpa.yaml | 52 + .../templates/image-renderer-deployment.yaml | 131 + .../grafana/templates/image-renderer-hpa.yaml | 47 + .../image-renderer-network-policy.yaml | 79 + .../templates/image-renderer-service.yaml | 31 + .../image-renderer-servicemonitor.yaml | 48 + .../charts/grafana/templates/ingress.yaml | 78 + .../grafana/templates/networkpolicy.yaml | 61 + .../grafana/templates/nginx-config.yaml | 94 + .../templates/poddisruptionbudget.yaml | 22 + .../grafana/templates/podsecuritypolicy.yaml | 45 + .../charts/grafana/templates/pvc.yaml | 41 + .../charts/grafana/templates/role.yaml | 32 + .../charts/grafana/templates/rolebinding.yaml | 25 + .../charts/grafana/templates/secret-env.yaml | 14 + .../charts/grafana/templates/secret.yaml | 16 + .../charts/grafana/templates/service.yaml | 61 + .../grafana/templates/serviceaccount.yaml | 17 + .../grafana/templates/servicemonitor.yaml | 68 + .../charts/grafana/templates/statefulset.yaml | 58 + .../templates/tests/test-configmap.yaml | 20 + .../tests/test-podsecuritypolicy.yaml | 32 + .../grafana/templates/tests/test-role.yaml | 17 + .../templates/tests/test-rolebinding.yaml | 20 + .../templates/tests/test-serviceaccount.yaml | 12 + .../charts/grafana/templates/tests/test.yaml | 53 + .../charts/grafana/values.yaml | 1315 ++++ .../charts/hardenedKubelet/.helmignore | 23 + .../charts/hardenedKubelet/Chart.yaml | 15 + .../charts/hardenedKubelet/README.md | 90 + .../hardenedKubelet/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedKubelet/values.yaml | 166 + .../charts/hardenedNodeExporter/.helmignore | 23 + .../charts/hardenedNodeExporter/Chart.yaml | 15 + .../charts/hardenedNodeExporter/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedNodeExporter/values.yaml | 166 + .../charts/k3sServer/.helmignore | 23 + .../charts/k3sServer/Chart.yaml | 15 + .../charts/k3sServer/README.md | 90 + .../charts/k3sServer/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../k3sServer/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../k3sServer/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/k3sServer/values.yaml | 166 + .../charts/kube-state-metrics/.helmignore | 21 + .../charts/kube-state-metrics/Chart.yaml | 32 + .../charts/kube-state-metrics/README.md | 85 + .../kube-state-metrics/templates/NOTES.txt | 23 + .../kube-state-metrics/templates/_helpers.tpl | 196 + .../templates/ciliumnetworkpolicy.yaml | 33 + .../templates/clusterrolebinding.yaml | 20 + .../templates/crs-configmap.yaml | 16 + .../templates/deployment.yaml | 314 + .../templates/extra-manifests.yaml | 4 + .../templates/kubeconfig-secret.yaml | 12 + .../templates/networkpolicy.yaml | 43 + .../kube-state-metrics/templates/pdb.yaml | 18 + .../templates/podsecuritypolicy.yaml | 39 + .../templates/psp-clusterrole.yaml | 19 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/rbac-configmap.yaml | 22 + .../kube-state-metrics/templates/role.yaml | 215 + .../templates/rolebinding.yaml | 24 + .../kube-state-metrics/templates/service.yaml | 49 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 126 + .../templates/stsdiscovery-role.yaml | 26 + .../templates/stsdiscovery-rolebinding.yaml | 17 + .../templates/verticalpodautoscaler.yaml | 44 + .../charts/kube-state-metrics/values.yaml | 491 ++ .../kubeAdmControllerManager/.helmignore | 23 + .../kubeAdmControllerManager/Chart.yaml | 15 + .../charts/kubeAdmControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../kubeAdmControllerManager/values.yaml | 166 + .../charts/kubeAdmEtcd/.helmignore | 23 + .../charts/kubeAdmEtcd/Chart.yaml | 15 + .../charts/kubeAdmEtcd/README.md | 90 + .../charts/kubeAdmEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../kubeAdmEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmEtcd/values.yaml | 166 + .../charts/kubeAdmProxy/.helmignore | 23 + .../charts/kubeAdmProxy/Chart.yaml | 15 + .../charts/kubeAdmProxy/README.md | 90 + .../kubeAdmProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmProxy/values.yaml | 166 + .../charts/kubeAdmScheduler/.helmignore | 23 + .../charts/kubeAdmScheduler/Chart.yaml | 15 + .../charts/kubeAdmScheduler/README.md | 90 + .../kubeAdmScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmScheduler/values.yaml | 166 + .../charts/prometheus-adapter/.helmignore | 21 + .../charts/prometheus-adapter/Chart.yaml | 28 + .../charts/prometheus-adapter/README.md | 160 + .../prometheus-adapter/templates/NOTES.txt | 9 + .../prometheus-adapter/templates/_helpers.tpl | 113 + .../templates/certmanager.yaml | 76 + .../cluster-role-binding-auth-delegator.yaml | 20 + .../cluster-role-binding-resource-reader.yaml | 20 + .../cluster-role-resource-reader.yaml | 24 + .../templates/configmap.yaml | 97 + .../templates/custom-metrics-apiservice.yaml | 34 + ...stom-metrics-cluster-role-binding-hpa.yaml | 24 + .../custom-metrics-cluster-role.yaml | 17 + .../templates/deployment.yaml | 143 + .../external-metrics-apiservice.yaml | 34 + ...rnal-metrics-cluster-role-binding-hpa.yaml | 20 + .../external-metrics-cluster-role.yaml | 21 + .../prometheus-adapter/templates/pdb.yaml | 23 + .../prometheus-adapter/templates/psp.yaml | 66 + .../resource-metrics-apiservice.yaml | 34 + ...resource-metrics-cluster-role-binding.yaml | 20 + .../resource-metrics-cluster-role.yaml | 23 + .../templates/role-binding-auth-reader.yaml | 21 + .../prometheus-adapter/templates/secret.yaml | 17 + .../prometheus-adapter/templates/service.yaml | 27 + .../templates/serviceaccount.yaml | 18 + .../charts/prometheus-adapter/values.yaml | 277 + .../prometheus-node-exporter/.helmignore | 21 + .../prometheus-node-exporter/Chart.yaml | 25 + .../charts/prometheus-node-exporter/README.md | 97 + .../templates/NOTES.txt | 29 + .../templates/_helpers.tpl | 236 + .../templates/clusterrole.yaml | 19 + .../templates/clusterrolebinding.yaml | 20 + .../templates/daemonset.yaml | 309 + .../templates/endpoints.yaml | 18 + .../templates/extra-manifests.yaml | 4 + .../templates/networkpolicy.yaml | 23 + .../templates/podmonitor.yaml | 91 + .../templates/psp-clusterrole.yaml | 14 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/psp.yaml | 49 + .../templates/rbac-configmap.yaml | 16 + .../templates/service.yaml | 29 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 71 + .../templates/verticalpodautoscaler.yaml | 40 + .../prometheus-node-exporter/values.yaml | 530 ++ .../charts/rke2ControllerManager/.helmignore | 23 + .../charts/rke2ControllerManager/Chart.yaml | 15 + .../charts/rke2ControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2ControllerManager/values.yaml | 166 + .../charts/rke2Etcd/.helmignore | 23 + .../charts/rke2Etcd/Chart.yaml | 15 + .../charts/rke2Etcd/README.md | 90 + .../charts/rke2Etcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Etcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Etcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Etcd/values.yaml | 166 + .../charts/rke2IngressNginx/.helmignore | 23 + .../charts/rke2IngressNginx/Chart.yaml | 15 + .../charts/rke2IngressNginx/README.md | 90 + .../rke2IngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2IngressNginx/values.yaml | 166 + .../charts/rke2Proxy/.helmignore | 23 + .../charts/rke2Proxy/Chart.yaml | 15 + .../charts/rke2Proxy/README.md | 90 + .../charts/rke2Proxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Proxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Proxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Proxy/values.yaml | 166 + .../charts/rke2Scheduler/.helmignore | 23 + .../charts/rke2Scheduler/Chart.yaml | 15 + .../charts/rke2Scheduler/README.md | 90 + .../rke2Scheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Scheduler/values.yaml | 166 + .../charts/rkeControllerManager/.helmignore | 23 + .../charts/rkeControllerManager/Chart.yaml | 15 + .../charts/rkeControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeControllerManager/values.yaml | 166 + .../charts/rkeEtcd/.helmignore | 23 + .../charts/rkeEtcd/Chart.yaml | 15 + .../charts/rkeEtcd/README.md | 90 + .../charts/rkeEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeEtcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeEtcd/values.yaml | 166 + .../charts/rkeIngressNginx/.helmignore | 23 + .../charts/rkeIngressNginx/Chart.yaml | 15 + .../charts/rkeIngressNginx/README.md | 90 + .../rkeIngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeIngressNginx/values.yaml | 166 + .../charts/rkeProxy/.helmignore | 23 + .../charts/rkeProxy/Chart.yaml | 15 + .../charts/rkeProxy/README.md | 90 + .../charts/rkeProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeProxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeProxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeProxy/values.yaml | 166 + .../charts/rkeScheduler/.helmignore | 23 + .../charts/rkeScheduler/Chart.yaml | 15 + .../charts/rkeScheduler/README.md | 90 + .../rkeScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeScheduler/values.yaml | 166 + .../charts/windowsExporter/.helmignore | 21 + .../charts/windowsExporter/Chart.yaml | 17 + .../charts/windowsExporter/README.md | 42 + .../scripts/configure-firewall.ps1 | 31 + .../windowsExporter/templates/_helpers.tpl | 216 + .../windowsExporter/templates/config.yaml | 14 + .../windowsExporter/templates/daemonset.yaml | 200 + .../windowsExporter/templates/podmonitor.yaml | 91 + .../templates/scriptConfig.yaml | 14 + .../windowsExporter/templates/service.yaml | 32 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 75 + .../charts/windowsExporter/values.yaml | 366 ++ .../files/ingress-nginx/nginx.json | 1445 +++++ .../request-handling-performance.json | 963 +++ .../cluster/rancher-cluster-nodes.json | 793 +++ .../rancher/cluster/rancher-cluster.json | 776 +++ .../files/rancher/fleet/bundle.json | 246 + .../files/rancher/fleet/bundledeployment.json | 219 + .../files/rancher/fleet/cluster.json | 484 ++ .../files/rancher/fleet/clustergroup.json | 468 ++ .../rancher/fleet/controller-runtime.json | 454 ++ .../files/rancher/fleet/gitrepo.json | 325 + .../rancher/home/rancher-default-home.json | 1290 ++++ .../files/rancher/k8s/rancher-etcd-nodes.json | 687 +++ .../files/rancher/k8s/rancher-etcd.json | 669 ++ .../k8s/rancher-k8s-components-nodes.json | 527 ++ .../rancher/k8s/rancher-k8s-components.json | 519 ++ .../rancher/nodes/rancher-node-detail.json | 805 +++ .../files/rancher/nodes/rancher-node.json | 792 +++ .../performance/performance-debugging.json | 1652 +++++ .../rancher/pods/rancher-pod-containers.json | 636 ++ .../files/rancher/pods/rancher-pod.json | 636 ++ .../workloads/rancher-workload-pods.json | 652 ++ .../rancher/workloads/rancher-workload.json | 652 ++ .../delete-workloads-with-old-labels.sh | 14 + .../templates/NOTES.txt | 4 + .../templates/_helpers.tpl | 459 ++ .../templates/alertmanager/alertmanager.yaml | 191 + .../templates/alertmanager/extrasecret.yaml | 20 + .../templates/alertmanager/ingress.yaml | 78 + .../alertmanager/ingressperreplica.yaml | 67 + .../alertmanager/podDisruptionBudget.yaml | 21 + .../templates/alertmanager/psp-role.yaml | 23 + .../alertmanager/psp-rolebinding.yaml | 20 + .../templates/alertmanager/psp.yaml | 47 + .../templates/alertmanager/secret.yaml | 35 + .../templates/alertmanager/service.yaml | 68 + .../alertmanager/serviceaccount.yaml | 21 + .../alertmanager/servicemonitor.yaml | 84 + .../alertmanager/serviceperreplica.yaml | 49 + .../templates/exporters/core-dns/service.yaml | 24 + .../exporters/core-dns/servicemonitor.yaml | 58 + .../kube-api-server/servicemonitor.yaml | 57 + .../kube-controller-manager/endpoints.yaml | 22 + .../kube-controller-manager/service.yaml | 29 + .../servicemonitor.yaml | 69 + .../templates/exporters/kube-dns/service.yaml | 28 + .../exporters/kube-dns/servicemonitor.yaml | 71 + .../exporters/kube-etcd/endpoints.yaml | 20 + .../exporters/kube-etcd/service.yaml | 27 + .../exporters/kube-etcd/servicemonitor.yaml | 75 + .../exporters/kube-proxy/endpoints.yaml | 20 + .../exporters/kube-proxy/service.yaml | 27 + .../exporters/kube-proxy/servicemonitor.yaml | 63 + .../exporters/kube-scheduler/endpoints.yaml | 22 + .../exporters/kube-scheduler/service.yaml | 29 + .../kube-scheduler/servicemonitor.yaml | 69 + .../kube-state-metrics/validate.yaml | 7 + .../exporters/kubelet/servicemonitor.yaml | 246 + .../exporters/node-exporter/validate.yaml | 3 + .../templates/extra-objects.yaml | 4 + .../grafana/configmap-dashboards.yaml | 24 + .../grafana/configmaps-datasources.yaml | 81 + .../alertmanager-overview.yaml | 616 ++ .../grafana/dashboards-1.14/apiserver.yaml | 1772 ++++++ .../dashboards-1.14/cluster-total.yaml | 1882 ++++++ .../dashboards-1.14/controller-manager.yaml | 1196 ++++ .../grafana/dashboards-1.14/etcd.yaml | 1229 ++++ .../dashboards-1.14/grafana-overview.yaml | 635 ++ .../grafana/dashboards-1.14/k8s-coredns.yaml | 1534 +++++ .../k8s-resources-cluster.yaml | 3088 ++++++++++ .../k8s-resources-multicluster.yaml | 24 + .../k8s-resources-namespace.yaml | 2797 +++++++++ .../dashboards-1.14/k8s-resources-node.yaml | 1026 ++++ .../dashboards-1.14/k8s-resources-pod.yaml | 2469 ++++++++ .../k8s-resources-windows-cluster.yaml | 24 + .../k8s-resources-windows-namespace.yaml | 24 + .../k8s-resources-windows-pod.yaml | 24 + .../k8s-resources-workload.yaml | 2024 ++++++ .../k8s-resources-workloads-namespace.yaml | 2189 +++++++ .../k8s-windows-cluster-rsrc-use.yaml | 24 + .../k8s-windows-node-rsrc-use.yaml | 24 + .../grafana/dashboards-1.14/kubelet.yaml | 2256 +++++++ .../dashboards-1.14/namespace-by-pod.yaml | 1464 +++++ .../namespace-by-workload.yaml | 1736 ++++++ .../node-cluster-rsrc-use.yaml | 1063 ++++ .../dashboards-1.14/node-rsrc-use.yaml | 1089 ++++ .../grafana/dashboards-1.14/nodes-darwin.yaml | 1073 ++++ .../grafana/dashboards-1.14/nodes.yaml | 1066 ++++ .../persistentvolumesusage.yaml | 587 ++ .../grafana/dashboards-1.14/pod-total.yaml | 1228 ++++ .../prometheus-remote-write.yaml | 1674 +++++ .../grafana/dashboards-1.14/prometheus.yaml | 1235 ++++ .../grafana/dashboards-1.14/proxy.yaml | 1276 ++++ .../grafana/dashboards-1.14/scheduler.yaml | 1118 ++++ .../dashboards-1.14/workload-total.yaml | 1438 +++++ .../templates/grafana/namespaces.yaml | 13 + .../_prometheus-operator.tpl | 7 + .../_prometheus-operator-webhook.tpl | 6 + .../deployment/deployment.yaml | 143 + .../admission-webhooks/deployment/pdb.yaml | 15 + .../deployment/service.yaml | 58 + .../deployment/serviceaccount.yaml | 15 + .../ciliumnetworkpolicy-createSecret.yaml | 36 + .../ciliumnetworkpolicy-patchWebhook.yaml | 36 + .../job-patch/clusterrole.yaml | 33 + .../job-patch/clusterrolebinding.yaml | 20 + .../job-patch/job-createSecret.yaml | 73 + .../job-patch/job-patchWebhook.yaml | 74 + .../job-patch/networkpolicy-createSecret.yaml | 33 + .../job-patch/networkpolicy-patchWebhook.yaml | 33 + .../admission-webhooks/job-patch/psp.yaml | 47 + .../admission-webhooks/job-patch/role.yaml | 21 + .../job-patch/rolebinding.yaml | 21 + .../job-patch/serviceaccount.yaml | 17 + .../mutatingWebhookConfiguration.yaml | 77 + .../validatingWebhookConfiguration.yaml | 77 + .../prometheus-operator/certmanager.yaml | 55 + .../ciliumnetworkpolicy.yaml | 40 + .../prometheus-operator/clusterrole.yaml | 109 + .../clusterrolebinding.yaml | 16 + .../prometheus-operator/deployment.yaml | 204 + .../prometheus-operator/networkpolicy.yaml | 29 + .../prometheus-operator/psp-clusterrole.yaml | 21 + .../psp-clusterrolebinding.yaml | 18 + .../templates/prometheus-operator/psp.yaml | 46 + .../prometheus-operator/service.yaml | 57 + .../prometheus-operator/serviceaccount.yaml | 14 + .../prometheus-operator/servicemonitor.yaml | 57 + .../verticalpodautoscaler.yaml | 40 + .../templates/prometheus/_rules.tpl | 44 + .../additionalAlertRelabelConfigs.yaml | 16 + .../additionalAlertmanagerConfigs.yaml | 16 + .../prometheus/additionalPrometheusRules.yaml | 43 + .../prometheus/additionalScrapeConfigs.yaml | 20 + .../prometheus/ciliumnetworkpolicy.yaml | 27 + .../templates/prometheus/clusterrole.yaml | 30 + .../prometheus/clusterrolebinding.yaml | 18 + .../templates/prometheus/csi-secret.yaml | 12 + .../templates/prometheus/extrasecret.yaml | 20 + .../templates/prometheus/ingress.yaml | 77 + .../prometheus/ingressThanosSidecar.yaml | 77 + .../prometheus/ingressperreplica.yaml | 67 + .../templates/prometheus/networkpolicy.yaml | 34 + .../templates/prometheus/nginx-config.yaml | 68 + .../prometheus/podDisruptionBudget.yaml | 25 + .../templates/prometheus/podmonitors.yaml | 38 + .../templates/prometheus/prometheus.yaml | 472 ++ .../templates/prometheus/psp-clusterrole.yaml | 22 + .../prometheus/psp-clusterrolebinding.yaml | 19 + .../templates/prometheus/psp.yaml | 58 + .../rules-1.14/alertmanager.rules.yaml | 305 + .../rules-1.14/config-reloaders.yaml | 57 + .../templates/prometheus/rules-1.14/etcd.yaml | 461 ++ .../prometheus/rules-1.14/general.rules.yaml | 125 + ...les.container_cpu_usage_seconds_total.yaml | 43 + .../k8s.rules.container_memory_cache.yaml | 42 + .../k8s.rules.container_memory_rss.yaml | 42 + .../k8s.rules.container_memory_swap.yaml | 42 + ...es.container_memory_working_set_bytes.yaml | 42 + .../k8s.rules.container_resource.yaml | 168 + .../rules-1.14/k8s.rules.pod_owner.yaml | 107 + .../prometheus/rules-1.14/k8s.rules.yaml | 237 + .../kube-apiserver-availability.rules.yaml | 273 + .../kube-apiserver-burnrate.rules.yaml | 440 ++ .../kube-apiserver-histogram.rules.yaml | 53 + .../rules-1.14/kube-apiserver-slos.yaml | 159 + .../kube-prometheus-general.rules.yaml | 49 + .../kube-prometheus-node-recording.rules.yaml | 93 + .../rules-1.14/kube-scheduler.rules.yaml | 135 + .../rules-1.14/kube-state-metrics.yaml | 152 + .../prometheus/rules-1.14/kubelet.rules.yaml | 65 + .../rules-1.14/kubernetes-apps.yaml | 568 ++ .../rules-1.14/kubernetes-resources.yaml | 282 + .../rules-1.14/kubernetes-storage.yaml | 216 + .../kubernetes-system-apiserver.yaml | 193 + .../kubernetes-system-controller-manager.yaml | 55 + .../kubernetes-system-kube-proxy.yaml | 56 + .../rules-1.14/kubernetes-system-kubelet.yaml | 379 ++ .../kubernetes-system-scheduler.yaml | 54 + .../rules-1.14/kubernetes-system.yaml | 87 + .../rules-1.14/node-exporter.rules.yaml | 188 + .../prometheus/rules-1.14/node-exporter.yaml | 801 +++ .../prometheus/rules-1.14/node-network.yaml | 55 + .../prometheus/rules-1.14/node.rules.yaml | 109 + .../rules-1.14/prometheus-operator.yaml | 253 + .../prometheus/rules-1.14/prometheus.yaml | 707 +++ .../rules-1.14/windows.node.rules.yaml | 301 + .../rules-1.14/windows.pod.rules.yaml | 158 + .../templates/prometheus/secret.yaml | 15 + .../templates/prometheus/service.yaml | 80 + .../prometheus/serviceThanosSidecar.yaml | 39 + .../serviceThanosSidecarExternal.yaml | 46 + .../templates/prometheus/serviceaccount.yaml | 21 + .../templates/prometheus/servicemonitor.yaml | 97 + .../servicemonitorThanosSidecar.yaml | 55 + .../templates/prometheus/servicemonitors.yaml | 47 + .../prometheus/serviceperreplica.yaml | 54 + .../rancher-monitoring/clusterrole.yaml | 135 + .../rancher-monitoring/config-role.yaml | 48 + .../rancher-monitoring/dashboard-role.yaml | 47 + .../addons/ingress-nginx-dashboard.yaml | 18 + .../rancher/cluster-dashboards.yaml | 17 + .../dashboards/rancher/default-dashboard.yaml | 17 + .../dashboards/rancher/fleet-dashboards.yaml | 17 + .../dashboards/rancher/k8s-dashboards.yaml | 31 + .../dashboards/rancher/nodes-dashboards.yaml | 17 + .../rancher/performance-dashboards.yaml | 18 + .../dashboards/rancher/pods-dashboards.yaml | 17 + .../rancher/workload-dashboards.yaml | 17 + .../exporters/fleet/servicemonitor.yaml | 53 + .../exporters/ingress-nginx/service.yaml | 27 + .../ingress-nginx/servicemonitor.yaml | 49 + .../exporters/rancher/servicemonitor.yaml | 58 + .../rancher-monitoring/hardened.yaml | 147 + .../rancher-monitoring/upgrade/configmap.yaml | 13 + .../rancher-monitoring/upgrade/job.yaml | 46 + .../rancher-monitoring/upgrade/rbac.yaml | 131 + .../templates/thanos-ruler/extrasecret.yaml | 20 + .../templates/thanos-ruler/ingress.yaml | 77 + .../thanos-ruler/podDisruptionBudget.yaml | 21 + .../templates/thanos-ruler/ruler.yaml | 189 + .../templates/thanos-ruler/secret.yaml | 26 + .../templates/thanos-ruler/service.yaml | 53 + .../thanos-ruler/serviceaccount.yaml | 20 + .../thanos-ruler/servicemonitor.yaml | 82 + .../templates/validate-install-crd.yaml | 23 + .../templates/validate-psp-install.yaml | 7 + .../values.yaml | 5431 +++++++++++++++++ index.yaml | 144 + .../rancher-monitoring/package.yaml | 2 +- .../templates/psp-clusterrole.yaml.patch | 2 +- release.yaml | 70 +- 584 files changed, 105273 insertions(+), 36 deletions(-) create mode 100644 assets/rancher-monitoring-crd/rancher-monitoring-crd-104.1.2-rc.1+up57.0.3.up57.0.3.tgz create mode 100644 assets/rancher-monitoring/rancher-monitoring-104.1.2-rc.1+up57.0.3.up57.0.3.tgz create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/Chart.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/README.md create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/files/crd-manifest.tgz create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/jobs.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/manifest.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rbac.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/.editorconfig create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/CHANGELOG.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/CONTRIBUTING.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/app-README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/dashboards/custom-dashboard.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/_config.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/_pod.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/configSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/headless-service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/nginx-config.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/pvc.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/secret-env.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/statefulset.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/config.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/ingress-nginx/nginx.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/ingress-nginx/request-handling-performance.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/cluster/rancher-cluster.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/bundle.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/bundledeployment.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/cluster.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/clustergroup.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/controller-runtime.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/gitrepo.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/home/rancher-default-home.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-etcd.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-k8s-components.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/nodes/rancher-node-detail.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/nodes/rancher-node.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/performance/performance-debugging.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/pods/rancher-pod-containers.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/pods/rancher-pod.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/workloads/rancher-workload-pods.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/workloads/rancher-workload.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/alertmanager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/extrasecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/psp-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/psp-rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/core-dns/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/core-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-controller-manager/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-dns/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-etcd/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-etcd/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-proxy/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-proxy/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-scheduler/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-state-metrics/validate.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kubelet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/node-exporter/validate.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/extra-objects.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/configmap-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/configmaps-datasources.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/namespaces.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/certmanager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/_rules.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalPrometheusRules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/csi-secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/extrasecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ingressThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/nginx-config.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/podmonitors.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/prometheus.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node-network.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/servicemonitors.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/config-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboard-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/hardened.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/upgrade/job.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/extrasecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/ruler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/values.yaml diff --git a/assets/rancher-monitoring-crd/rancher-monitoring-crd-104.1.2-rc.1+up57.0.3.up57.0.3.tgz b/assets/rancher-monitoring-crd/rancher-monitoring-crd-104.1.2-rc.1+up57.0.3.up57.0.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..29a0f8c2e6c4783089fe5696e19df004b829743a GIT binary patch literal 299360 zcmV)5K*_%!iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMR7Jd|JCIF1MtA;}uIElbSUw^FvUFWE}6jF}t5m>IL!_kC-j z$W|y>BO$w#C|jXONFhnql%)v&&!Fx5e7E;~pWpMm|KIO9pFX+9L zjzKtscu6-b21Uf;Q5Yvl1Re>~bcW-JP;a;!n&N-nq@<*z6y)SK|Cf@I`tAQx@^bPN z(sBxlGSbpg(ozZ(QquDBataiH)IVYFzicD|5ss&j`XA5r+r|C=q`@&5ED=sbVKD?1 z2mm19L^vAj1VzA!L^KFRVPOamPeeJQK%@$=y;pu0%o&A5f*2J*fQTo70>2A_fxCeO z92^0v09&0uZ#sfEfe5<&?ss?)4Z;baB>DUAMEqvMLf|-*C5R`WuoxA;N0UIpKgj_6mHx}gOZ`^=6%`f#qyPVk zCJ1POj&Ks12yBV`4^)G^DWDKRz}y){08j(~4p^xh8b~@~@osP;5kvxxC^Se0&;k)? zI35H%;CK|=0SyuWA{KA}0UVq_0FeL+L&O5!SP~u}f^Il8oCp%25Qq~R>i|b@+aDqt zBtHQFj)2>G000m}#)VV?9N}mJxY@a7Ft&*|1)Wd?BHmjC5Db{ct}BIpwbWs28#sEKs1OTV(}_~uRjDr#G*m`wp|3QZT>f_ z@c*d&f&PD&4$RIOMB_j_0ZPREcTK@x=)bg#qU^8wFC!%_C-oow|5r3Yz;xTILwtQD z0VL>%!hnFlmUh`8x75oHzpY;afTX|wCLqcYfLd-ji7oB@=CFTPFTa0=!=o@nM?gTB z02BCcApkMRe|Uxn0w+-zB-WDvn2~TeES^YS3``vItrvkN5r`lj@I;}}033+~w$}v^ z;V2Ao)8wI17?Kx2wgUtS2pPlyL}NX-yMHnUWOL?;#k-=ha3ldhU`c2)C_q3sgGdq@ zM3Tpe&R7EYGe(Fwj7;Xw4Bi}-B#%n|gnf&@l7K5nHf29TAS{6#Qf*pjmF;iI9iXEl zh#;x}0!CP~Z34*MSR5H2*^&tyb0@*k0>5(W56B>F+rM9AOaR?HK)fc_4M#TW0*9rc z(sGhg0svGKj)Oa(&?q7bBtZ2^4&Zk51=U6ow;wn0IDkYr)Rp7_;xQoE*l#XIWs5mm zWT6S*Cgh(2{hrkS-Kz6{Mf*$p?}ByspT>cIVgD6mo|0&AJ$^6Iu|0~+hd6NU2 zh;W8^NJCsv7^DiIhjoCsfkZeGPK1-AsjcM5*B5|pRR@gVZXn?AFNwf|a3Z)3u$|zM zp-i{(woSOrZouCk0sv^Z1BfPUMRsr;?sqUk0N^)iDF6sKgJ?G>!5QX^#kvZp05~jx zD2dui>?F3{NH`}v90_i};jw76103P{k6%k7K{QANC2?3Z3gIoF0yuz&_L`%8wA+t2LyQ5g7}}vmfT+L*4oH6gz$X~0Jb7>GJ)SX_9r0*6!7;aFJ(JA z*!&jyBa{dH{Z;;9mE+`ANNWB8 zIKNmIw~kbs}x_wg-({W5&$ch;tb#en<*KMi3Pt#%Tg4I;;H53COoE$@ZG~O{&J2<8Tz3g#U)) zH;J4tfWjcqB;+4OI#9q5z@RWl5JLo%er%xM1h_vV{S}CawSv2$e;@DN_x=geFGh;X4!I43MiGc2Hh+HZg(Kjg+sD+v<{8cp;sI-st(;H9<`;4|`;p(WqfHPL zNZ170CY0Q)MMwnjmoGQX>DFTZn2A3{SieoE|7&vAKw*$5jMM)?abwY-DdFD zp%DTA|F{+Znj!+p!39JRH@DT+_h!FjvVUt)f5*N3mf7D~`kVRwYwY}I)iA!JB$N4H z+0Q>OX=^jXJfs~!BK+I(Ot46^?VY@pMg6;1_^oPz!=pSmzV98tZBDZQvLI z0Y}5P?uvbbaK;jdMj+7>i+BCCQ`bcE*A5e1tsfmfsur6hk}a7EAYw}HRmXUXKmdRM z8jw@4KOQIHaA?pC#1P@=AI9%Hs3D0!+yocd>V;#Re$JZQP#6_J`g>aB2KQ0{6y)V) z%Aj0uZAo2SZAS4zfY)$#I z8vwmfgw0c=?O!zzev$*>``H^3@I(=ve_s;Vt=@0UC1L>rNCfQ3fjAV02MHkF1GK=n zVz8c=<0ue~B&cr2iuQ-LRz)C!II^_xpgReL2ax~)g+YK4n+H^6$pO~-->(P)NGyV2 zBkD{f;s`1**spg7$zw1goZw1;ZPp8rl0U{_Tjc@C?^*b7J0yRq7{CM_$#hEM@FWZh z<0Q5zWjJ8HxrEJa498Zhb^D1XNdn4=03&m7QsyBU7o3qR{1 z_8Y-lNKn6Q60&{(a121^@Ah4;-{0W`5C_M@iQv{|-acIXCvD}ww~fE0vrtDA298Gg zfOvwcwT=Dnux^XGe-7^uXpqbv+)uw37bC@!GmsC4-oke%lye!|DbNm$DDNbCaZ@C5%2^7 z7ZQQE+;K%G_$zw@G0K{y%7m(~O0r+OnO2UI=j^W830>8C9 z>rGl@ZT|K;5qRWZE|DA>Z{gkC{sh?1jueYQdv6vZuz2E@uiuv2W*$JsN1jdsa3B$X z1k~H`o8aFxp3I<4fx^*0z9M1JC|3~hbl%*3-#3~g9I+YsZ~i9e-yIhHKhypo{{Q_$ zxBuoGe-{5sOUug0EBwRnf23sP{uBTID;i%c-PXg#biJ$Rn~FP9FFU`mc^$B7mn}@1 zPC4T&V)a_=^UfXG|;jo_e2GNKz-F;X@=&BLz`s~ip^$nfO+v}4@1=Cre zgfqjOBlHJzhzB0(r#mfwiLhOUHB6t`cn7=dxBj&qHZZj@`Xv^2bz`Uh1>LVLeH&(S zmrM7GXlwJlKI>%uq)=CQVL;!s#l7+4^@OtY7Ule=sKhWttJk}^eUIL}uIJ$6i(giP z$v$1|5FL4c;t52YGq)L?E@hN#V)rE<`_5m6>+F&H!fRJ?>Q?R{INerXUC1I#`tBN& zbk4N%{MU=tr?W*SKw+VFShPKxaSLb4(-6J37-PJEdyMdrFVy^J_8K@Wx@N6MU*Li| zSsnlEd^~2}O`iW^jHB2HSM}<`sMpu8pXP4|G&7>=xE!afWH~bc^WpUL%;FJ}QIp6u zq3CNa53MY&X3j=3oYq!NLOxU?ZSZED%kK-6CJ+ZQ-`>bMwjq68BVh2;Jy-GUfV>Xz zb7fEFCmLdv-+Y{jsbFdJmXYUUE>E`L9Jj#+hFs7&rE7E{pt^}(_o#Y{QSNbtQnPBW zb7lB*8Qkfu>0>Ya^kI7LKJM#l?@kuk30~S0k2|ebFPHW}Ga@R6HFzvNGuCJQG!vh| z9VKE-cJs~SXS>peQdsEazl!U3hb6xie9wQaIo&8vNUZT#4CKK3O9^{*BYG6ZX`Wm- z1<}Pz7rQ5&kxaXJvPbRBUb%o;g{mF$0iX8j^)6Lvmdfgy9s;8b&Rb;(}j{WeCM1h{vil-W*RUaZs<)N*6*~b_MoTg>zk5agGqa&Fw@}Uh=?p_@+Rl z&uM&}R(js8%mU}(XXm4&eI6*OmnWBAOoczrBjz53d)u;^L(ea$=Umj2fu1uh1^VMN z%UH%+)AbTsVlE1vc^ZG9zKh2`JvRg24r@}6wX8PhHr07$i}jD$X{okfK#Qhv$wD@! zSF6x2zrWg#L^}I2chTWy=iRHzy|E5=s9hSJrG;Ngj9w}`^G4{{X_?bHYv+ZoXP?Rl zIAMqv@2%M=6woNR!*%^~s&Ik7b)s03(EjV6sOp=PTWuZGm+n%y=wYS=^$C?Z%-_Glxi{)-VPndn zyu~**J=VPMJtnKd3)3v#V>uprT7hYS*49+Qwmp|;4!R_tsaH7OM{o$x@|0`047C$z zRlo@-aw;dazT7XXy&Qaskx(}ewczo(*dedcc@j}jeGvNm{c-#`;YO>6v(L|$Q?{De z>W;S5r(L9yRu*}=wv(`^ZG@}QGAp=ov_>HNh_BlXK`>hBDfX#@2Qqo~gliv-kq&_Ykrj1Q5J@>pByomc;tFwQpaJM$qqx<~VszDy36h5!#w zJn^cRU3iJI2*$K@vwhAzmLZr65im7KN$~KAUl9Q@tx}I#Ct2XRtwB!DYsar2vCzvm zeWbO zOl_CE<{kXdm7gV9^2UVt#l-e9*)pGlw#L&nR$tW)<;w5>N{;p zu>wD@VL{@Dz;)cP_ZZXNhK*ax+#83!o(&@8s}0hO?8;X*udj~C70+#UHSHMtO0D{u zCjU&43w06g(T?bS43En)mBkYE7l<*k>R0*42i?=1+g-I@9kejzrZpD6)lw#{QmFVM z+EK`Q=8Y!Nj4wYeg3X*hLEto2~))f7E~^Zt~h4r@6@w5^+08H|Cbq_f&9TJ1o{`pHDKsvKoyn9#_s^zhU*?DJQyRaR%+ zZr<2YDCBpjAGjVq{@is$XRvXWcl4;_ z$Je8E9Sem(>A2u}+f2-d7Uk2TVNAVWi64IL zaI^kAW+j5H^kl;%FpFNa^xgC1!_)QkJ?v!xqg0O9SIZ2iSDw*aJCFfG=W=9fLacT&#beY{1MnQPF^Lr-@+{8FH^pX3_4>)xd|%W}6qvMy9(Ia+j_ zKP`w(xD&E_4lJ}QhJ`hqIeva!uYU0$M_^Et%aIq@9aeh1M;6gl3YC4Z^3Hd!&N<%8 zFBkmqq}1k%@clY>2~_Yp3@YAn8Dmcmz4XFe=BEBaIlQ!|bf>ED8%~vy7rU>%e|=JU zMDBhHNIM??j`m0)CI3T~SsvWfK@qH!@i=Yd#PFBaceIhVH)%U(xQ|fjNeGWL)2}>w zmTULSX+th!)m!htnT-nvC62yHcsBix2Nt;QWN{&5!wPKAaF;I(`)YN;G{H~fbEibS z!CHg;u1~R-gvKKqRQXf`=a-#~8;*FdEFd>tdoT5uWgPv)c(}`fPSr;5(nB7o@YO?m zgZz1lxaP!p5K4s#fzdL-kqbd94`_Vmo6`>-6z0Q4b4Ojm#WcBPzr6E&G;G(DN>Q=y zM9FP%G=7n}iqzPH%QPSAOrpYK(5^Zhp}O)LGpv9vmZO^282 z-mUdh#n>OC;mUDM4 z+xsUQNrTp<3$A!>{Y=MKdX9&%3H<6Z^7q7?zHnqI_No=MLk>{G%kOa-?$5Cvr8mk_ zT5IQ9?dKR(f5zC}dI@LR@?^So&_%;bUC8Q6Mi?8t*bOCzU5)IQUhW@~4$zP7pE%d{ zr3JmiJzKG-J9j0J@)L1wc6x|j#n3DbZb>gwBqMi2NWZx%-rO&b)p~7QQG?B}f%jZh zW>t!#&&mpGhh>)ngF!OIP_^3g^vr{0>>S$WlcBt$?+U`6_{+;Q-lC3pcu_+`StpX< zN8Fi4HK$#8R=&+$xwQ0DY5IX3*aA1y3qxw-wf4oua6=i}cH9ul`TL3KvPK(4@y9Xe zo{iT!8{ulMMiw$OGRGD@$+rA@)#a@0Eo68ZI=aO-mu^yLEu$nedIe8mi!N<4)N|ec zp|7H-)SRfdtY(R0pSbWIyJjHuFf`btQCAP5OFOFO=FDgM_+?@O#`~y;Q;$w75SJI9 zRVP-(O}(>NtTau(K9L%2MeEKQsnMJr%Hhr#w^yEKs>0!!`la?qYRMcVW)UO710$ik zY-vZRl&JFe^#+H$xWCVp9h$s9Y}Z6o_vsV7l6m{XcA;tEA%nqoL6u_k%wxMN*d~gY z9uj6moX*xF+>G*1MBo{ko6v0yYV`Y%TD#OC>cTb`q*rdM9X-;m@#&FBFh{#jl#HP&JG+B5+0QV;PN{VsVi|a8E?gP(GDjD#!SH~ZE;i(fRD8LYjYoR~S z&`jeHxBF3+^ZmUNmzFqEH3uW#nH-iKwA|kxPqA_=tdlpyM^vdKa=tUFFm&QAH zioy)igXr0z@DO3WyqDUU23AS8YxNUMixt4^u+KSb9-0kU32U9aRvk*;_RcP3V_Zf< z;Q6Wtd|cy9*77;jPhwkG8V*^&db>N1Q|sx=w{~4>5fa_)3|=EFq*xD8BI4pkQF@(~ zpRQG3_JmhUpNo_84{ayO7RwjX(q=iWenn%C+m7lCnHtaXxWow0LhXBK6M=c|CiSm* zCvJMjuE@olj-(S|cXR5>bPgqkPEcL}AYv;w68uw3*x&4RLuFo@>63e9a*SU^%U?=R zyP-{DV8HR1b$^lQfXA^DO9uxYMjSizK}?)E*dssm&fbfp%b8$_OXnk{XS{-;Hj4cM zm7j8gD^WGgA0s(#2~s&Sbp+7(IxL+WoJI^UOyL5J_`X;n=)zPaPh^ZZFcUdp!J3a$ z;khK=W*GxI<#5WdbYM)5ju{Wh9NQ7GVdFi&tK;?t-xl_nkT5MUBOg|ZB7b!yba=qR>cyNP7`V?+8a);FF`AhwwgAHi~v~x=$@Eh)Z ziI}vwUa3Punibx*fz7kKZyM;bB6QClF}Ok@r4&NRaE>jAVIVA0IP~7|#0?0;IqfF< z`@?j!_Dg(XWwfE!(+VOnxGs$g*`s5VNlCA#@|yOIKjLJo2}^Uc8kEzNyBu4>=g{{t zNutsyVe$#Ol-;`VvX5Swq#>;t|sqo2A9o7BtG0vrwewHu4iMro+Jd56+U94C=>ctEt6hf^{YItDrE0_VHOGN z&lO;;ak#2)+>`?`Athu|KTPCiLe0gZM(5Ru2$RX9+6irFPuaZN{+W)6Ij4N*b{v`r zGr${^x#`U{n8IeWKCBnAsXqXh7ekk4l;aL6AMInma6?;q;01rrUB`NkZW<-3&J+!A z7V}52ijPr+jqgTKYY(Pw@s4WtTUaBj@KLk~r|3Y@(FRJBCcc=@w|4BE?~9XcVq|W2 zu(Q2-m$P1Z^{H&`(k!dov3ZIQX2ZGnjQQw~u0ivL-#?5mo{FD6S!DT=KedltY=kaX zOhlkXZqXE;TG&)RyP7lVLl{szyEdL${xachd_Je0!$9v!5#!vxFhc5{VBGDS(n;wO zqJ>%&S|K{QdWthJrqijHIa_u4cUT_0n>AdC{8G<5THrSG%)0cj+@0wReyhYIpAJ89 z=bezT6M1@j2JkWYx|l6`>(bSy`61$gT<3<7PMK@q+KPCB znyPl+ty?@1XQs5UDdmjk);mnv>S_$`XHSI&7liDkw@XTxFiN1&jk~NX)1+I>JcX?} zc>B;ty(0p&1ufE-U0!3#%qLad+$O4%2u}+7d4?XNWE6edE6tN#`wu>v==D-KKBj6( z<(OXbb*#ArWUFEm_U1bzv)C4?Bph7H6Ny!tZPD6Q-C9*?9cMjcA6J*pd0B@iU5h9F z&60B|MTHJm#3|LLNm@9J{{A5ymesrmi-+{jx=1~Uvb=acgi}LrZ%IF+R~>qv=`~(Q z*@Ti>|M=yVjoqa;>wz@(&u;fiKu;kXL{j>r$e&F=9WI4{3y{?Gr z-LZ#`*ORO6JVwbfl<4$f26ZYzPc7}96@RO-Cq32;bFjX;#`0G2>ysMqp3c~Jw-DI$a)l=xJ`_(O2%H{? z8W=N%^`iLWypzwyYoS;0*A0ji8|b|W$NLE4%xt3DA;+pvA+-&2_89*P>pr;?cDwJz z#+qk29IMds3P$Dh>hMcn){{3|(R;De_Wr2X*U@II=WL^m{KCsNRQ!8|yuy7cMoTIB z%q-*O)h8>wMXvX|%e~z75$@6-EJ)46KTJodtrMHTG=YA8{?TTv02TWlD|D>Uh?|7a zaCU24&@jG+P`n-fJ16JaY61{e#+! z7-l-dbPeYG`kt63A6P6Sn~=SitDc^|EZ6l4RRsbD*88S0cMg4mB@YC!yjutht>XCG_P!}!UCF?z#T)()&bIc* z$fFMk#_DN5aWweas2m>3$Jo=Ty$G3+Ic$nw3ivYgcIwp8hD57zfB%*p`H^5euTJz1Ps@xECC>dU2 z+V+T*fr=JPuV(nn+1Z=>9y0Qj7H8JF`JbWQNz1|WKr5615^NXAKoxrZvbdBIFH0y* zs31>Je_(b8FT*)Z0dLLy+lFlR<+*zIoMZ3A<v`*c9VJ8jBI*yB)LLn0hFNS;9BwDvlyHpKs-^?P(`k-)bv|j~%NE8XJBJmu=nf0!fF(lSDe(7V z{`&^_liNDVZmWoO&&?!V+RV$Q43j1cf~Rel-gSf>q)-nDJfFx|v3G&R31(38h<Df*t#<7$=wiB3{d}O3 z>9oc~$Om^fDs!5(lgF5ilvt@oZe}=MVRRMuJ7~_yRu7L2A{x(Kb-IxpAClnKHr;Tr zIit6BinQd5YwBt9%+ri{Tk90*HN(a)pL=jh?~r`%vl=Q%M{PY7h<6WHS9dNZAwZ^K zZp|pUq%k^2Rm0^j()(OIAUu95!5j;*vEjch$vuAR+98#aLIzP5i<1{Um#TPIjpV4# zJjMEJC{=h}vOd$O2+5%3n6skh0igW+#&1A;KXP#d5<> zL->(jzU6w$r5iBmx0Y2&>5|Tqh}mZ`Q*PLalHJiCPVBS^bBW)OvRBD z6qz%~_4VH6-13RCHa_U&$BdYZoWY}1zlhz z+YD;{+sdn7N3K z7Bc~Er_O10!Qgds&liZ zY&J>RcOGcA2pmg{^h1RS-S_?w%3si@!`=_^xe!I|v~p&rDl|b^w@^ZBdZvsRVBl6p zipIp8pYgsP&fa}eN0n)&>d3k(w+)xaI()?jVmU8#e0+mmq>@t-V{^BXtMFR?O%F4M znHNa*865%`HVz*Un0wQNtOgygJSb#CY-EC(&lDXE58F%KYyVPgT=Q){p zM5Ft{9C{TZj93!soYIGh(Jj8>*=m8O#oc2H4OVVG+-VvNDK*)bmm}*I@V3i9yqeIn zn%7%I(UxEj-OHP}Nk#+k ztfOCNvCpE`Oiqz)0-S{qRD5R_(=K=HGkOnDenYH}*^RKaL8JpSc}U z<4}F~r0pA)8VB{0H+33${ZHQHbV?LKxD^T7Bv@#0h1)e4#HOUeuV0)y{w~bcvYw*; zX6EI@9dxxCFW4H3H7VeI6ndGL*%nF%C?>$(Wx0jY0!p$HlC|;LG5Uql5L?T8vBB&+ zV7tBFoNSEEy4(xNyllbP)n34tc{!0b^K$P2HCQ1g$kwu}9pYm}qU=cy=8`q>uxI79 z?5E6=ex}dXlkBa848vOiDJ7qLs?awq(x;@98Yz?4wA-fd4^QMwW^wauV&7K7z1iwC_CdCm@k~j@ zT3@}-IW?KlzP@*w@l0Pet@&b416K0XN`9-O0u0iz|u|ju561nk1q+aaU=k;rj&|Rr*6%X}Nbu zF4YQ^k$1K2eH+nuo5A7(Ebto~!j*cRySVI+#|~aOum3^F=hVr=`T8wW@di<*i;ccl zPN{HP40s;8x&LH;vGNnw(=T5|j!Y!cp1=6|X2e|w#fn4r?o)jg1ClQS;7Np{iRTwF z0jjqjmEQDfrKYcM%d$OjUGeeCo-;ip=aRe)FZ-PL@DJA;a5AKhrP}tsS>{?Jp0a@S!(kuW9D9SVFniJUY(y zMByp3*Zkc}5wn_k<{GINu&p0+EJ9lrxzKUVp|aT_Cyf0Cn8xo;9vkdE0699ZKEA>` z#o~s#s?C!mML7I~VdKNYMcx8PIOj7K+5MQqk1SXqatesEZ*hXk9ZE~w&G-F$tpz_z zlO%7Ze1r)Sr4A(tzF5XE|{MYe?bE z9P0IyVwAq=nM#nCMu>zCQvdV8)izt7X^XqD5!~s`FNJ#W&xcCQr@d%U9T*u}w+ zAyDWFjSYlpj&#~Vj2>pxb@b|ywZZv`<-m=ZvbSZorZVQZj^>`tcy~7A)5FedUwr4M zr#d}uLkTI5a3Z-6DxW_frQ}epyhqS~CsChA%Wml$8Pe`%u1IA6}w0 z?mBQ-;!i_druto)dDNjh7_@*gTGwYkO?S~vf}sDV=Kp|&@(*-2ilVzD>f0^oMaTGyJMu1s`8F#;+dIxp+#|X^ z>tVoGJUs~KMZDqeqiR7CUrnqZU3r9cK8|5#yHi7@iqF*Ez0iW9Txe03zZEq$rZAG` zXp=Y{2tUI6WKVZBapS}O`~wHI1HXo1Oh5E~{><*oKrCQOi5ttlkUWLN2#rq{t0^Ix z4U~kz%j3tJ%cgYhyy=U%UNkNuLOCN?#{fD zxf2#W_Uv5QQlY4|j0>z&^OfbumB~lEY;pS+R#-WWFqNnU&l|QcH7_?5vZ+6iTwe%X zotn!Paa4S~hP#%3&h&Qab2)1U$67>jm=!;%lA-gQkWz~FLfNjGowe0xa#QQZ@o*t$ z>2{sifTP#V9+(wma@)|Z6@ksd`=`RWL?jg3T@v50B<&F)+(?Nre*7*_My5qS{OL8S zBU1U<=B|65b|3VlN_nS#Vk!AT*;UfjNG%_q-KoNp+(S+^V+P_gI_`H%qH<@cFYn>^ zPSmLZ7BL`CVzPd!Ip=u)o*U+QJ5HXEW{P`rLnuq*O{xUTzDk9ajDwO-&RMUe;gObt zbf|>xF-zb0#ZsMA*16(q;X*u9BWGXk>>5PL6*}u3G35HhQPX+B@J{Ryk8^B#*rk|s zh`Zq{S`qf++UNRbY`fd(#=0+eRc1ZHDqzp59@wM$RF?IFy*Y=9T(F>vy$19_js%p0 zr%GBt@tB|53VY7|Df9qUF2<$;aWgCJkTc@^!ev#aV>!D%LrB93KD*?bRxX#zEit?~ zXD}W&G%3JzLRvDTOyS8bq^dS=c6UAXJ+^Wx9L*v{RY(A-IFs6`pIW5y?$f#8*wha9 z6Bm+8VjXrbN#5}q5FAr^U-@$P5wQmzd*XFZJ^XBxHni?z5&Ic{fTCcKsijM~0CnpUqp!XvOxNp~yUr z;V3YWI;XD%v3xwve>s2euItx2J|tx40dse7@@HNb`|43&VLO#z{+^qw!48QTnWC4w zTPam&sHhVDo;dG}r;4J$r7{ zT7UO$-s%H6|JSvq&)mz-6S4z(Cn?X4Q|E?Z_g&i6Z?>Z-zmMCvPxAVX58~tcG~X&x zw=C(SNe1^nYo+b9plEBJ|J)q=dbUpHRl#DX<>gy4LsF<-gGc$*54#wMfkk)@1m+iZD% ze1vsw?BJV%yQNu2GvlwMShI^j3TOLP-dvQbTdh76(bXhI9Y3I7q;vCR{g8<|RYYY@ zcDdYjX_bs?^cM+kCk8{=8lx@u>iOwYi!1A1;xM9GI-brUd7IfY4%^q)Szi}WOBiAdTKGWK9J820eNdXWy!RaERpclK$W@+{{l=_u2i^D2Jd{nnvGr(%vlj zOAZ%}J47_5Pxvc$AyX%ga4zkMCkc9%%u!^=SC;L#a+jrFBADa4>}iv7d&oXL-wurW zpi@<*mt#edN~s{_LM&Z|>R5B;@Jo)p4aeOZ4?v9%oUtC%YiXiSPM$U8|Leo5-;}L1 zW#2Yho)ruJB!F)!x|>%-XATzHz2EK3@2=(vKT=U@kKEh2Bs18CC&1Ly_c=bc^by~03XCF(u`PyvXCg?xVpB>LGSyOLZqic}~_<*!mZ11kNRjTJ(+%M!y(zC#i&ZV$^HdK#UNT5rS^h?ho z#RZg1pLe%mGLbP8^pE}!X0gXTmgW5c|EsclYJeJ{WkpJGT9aiKz2|LRn@H`zLrURB zgj9V_Glq*Uj4flu-XFH>wnblirE7|{zl*VXdwbj1gr1jIZPU{Nh7mHw+5!;i;-)Bjer?!8rAHf%G_sj%%la?fFuBX6A-Ifjgy0uIa!nqgqR+^qDM8CL?NBAGA+4qTYQe)>-uTZ9m zKTIvlN7Q_d!cTnIj54E*ImlQzXpg8lQ~0DJ1*CUbl`(wMb-VDRcE$MwRrq96zG^*Ql_gQ$4}m8iMhj=cqQw4&y7l%nPh^d@x@u|y%ZY= zizMaJK`$(s(y;6Bv4n>2%H9qLalz$j1(5;4dm+7*l;mVH)Nwa2XPZUnHnHH+gaUTQRvpIGyJDBPri9)=byEt zcJ4-Tdj*nB&&qvxQmAl9BRt>p9RfXR<6k~h_+sBEL1ra2QBJwuuW+H$wf>?J?DZ@X z%4hc8TgBRsMg2{?#nZm`OeblnY`U$2!DjlWKGzH5KZiS!sIQFzO+cO59&iG-HHDG=uW3EEPqY6N5%;icFAT^>ny* z`~0ap;bxCfk}8i!sdB7J9lNuGsBW^Xp1)}2H@~L8W@_faA(u0Nj))z6pr+baSLw1d zJ{8hX6vN%h^Gv{mrztKS9J{2=G*q{k+1C?w$*X9%t-i9D?S(7W*Myt2tt&rRbwtPiIg=Z*Yz zFJ?%=dV*_DeaZAFqj>&+wJi6QZAxFy8NO7ET{64KXQT8ejr5+OyGD0c)&l*$d|ieG zY`mSz*q9xAjL#m@z1qEU=Jfuu_xE3TpyUtW`fQ8#6DuwX3Cf=P}^xTp1b>7E1-nxo) z+)0$}0wUP>< zWUn9A&q7Y`k2dx14jvgiRm=afU1RLhhpkm_os$w#lIO2yJb-y} z$5?Ce!xwp@0NGb#5qImGBdfATZ@qiEX5B=r{rZNFSk`{);UnIwTwcrx%1Q5>2E`$h zZo~kE!<`3;eM6l28@P5eaiya8TR&)J9~hZ-Gc%0k(71AIJfi#Qvd13V@QTuU3O6vT z=;n@=pfCoVmLMLgmLP^q>VY;5>wdQj~5hrWj~@PLY3W zjUxXRZ%|L4;b&1(yR0IK^28awgXM|g`16z$Z+TBru-#J_fm=|7;?LKct5-kLZnDpM ziTGBR_MOg|%)0CwR#EzEUE0!vrpt2QK$}lS#5Zcu>}u9lf4?qWg#P<=X)vou;+MMg zLGCa1e%Aep6^CMkyi?%HhlR6>5Yp^pjQWU)DwDa)^cx=TXXsyDczz%|4{v8UrI`FW zXS(?F`my2AtB=<4YvYU1@yV-2@S|$RFZRo=oa)k&d7^jQF!%i2V*8xl?MO>L`Pm>_ zz8z*(*EwI8^=qr1I5f31GJ|5b4sKeiTP#s{Q+V4j8Rt>hYjd%=`gG^B*iVjp3`}ho zVgywewGJn*7Mw+&)Td6hyh)L2xj-RT^to5Q=$`Q6fkRlH@qMxjPW2HA*t{_07e$}I z3w}%R%X1rSvfC-zFPTEiTqLzwVnR>PI~J+Wc0)4c}P<2S$8b8io)|~ zB3dkpcW;IkFL-F8I+`~;<=qQ+tvlMgWshifJc;MbHtvg(z2?`)uEU>y=IV(Pj$ZF8 zRr|_z#N`MT9DY>yO6v5CVE(}Mi*a5nozzB4mPJ~7J5lAV%FJCR)Lbz=6pY6#DehuF z_=nDM`@NRb%gmya+uKj!Ec$d=PHu1h&vjV2y}=Ozztmx~S|FKOI*eT=*?gH7U(Sc z^p`p;nJmU*nORZGC}+{9ySDMk?VbPIby%*Lu)kf0C1Y3nxegm45YIHQ#NGGGf#?=o zup!M~+;oozCi0!U zo0$puk;R;`lKu5QoVpKX1kYug?wdyPPa>jDQ_na6=@tq5%Bl`9<)->p4HhdyxeEn2 z8+G>7aeticf7xWde}h@o=z`UK#qJrxWJo9HLC2^%nCzM4@~a+iojN04*w{?$C%RS( zvVIImGm<=-$G+Tk<836Hy8r&uT_Fw&2{Fi)diX@GUF1GhCbrY7zH#$ih`Z8iY{|jrN<_+W@?Ip)VCRNMN4hkw!GTi zTSm>6!j?ykzQ<^h#t$La*4iBxH zc&di!Nh#eMrr<26WYXUme7{?tjf3ZW)cuhAEZe(qr?#R6)6noH_Wj$%KY&r3Vn>h7 zc|WQ1#aeEzCk1ss$~?e(>4mh%mVNyy5fb%D_K`>$DTa-Uw4x*(a{a><&hUGQlDxJz zhsUlmnce=n)g_!3Sv>qiM_tK~ zS%f}oSEJ2{R_^<1%1d4;(&^{s>`t7?8ZD(npTtWpwb@O zpwb?bBp=pH2@<18lF$B33HdYZp;HbIv>q^qFn=b!1cIg<>do(yZfiH$kJOync0Fg< zr*bBcRq3dP)oaerA81bw95|~bnTPLq;>a9;KV;@e~WC`2t z;BI_XFuG#nDdOEMs_EG7`_e^2Ah*NpoZz$py2JPNC9FXK?d`y}%Cw#dJwD0~xod5Y zPBStwjg~<8M%u6!p4L7p;I$U7X2FXX6Yc-%+Bns5q4< zd#0DcOnR*VETJmDB3^yb2-3>a?#3ndRn7Kv@a<1WATZa2h(Xb0k;vpn{=R~Ft^J0s z)Q1Jb7f7=7hkdKESDBX`Rs2I{N6!$ZML8$$qlC~m85V)F#R)VrL>fTTmjXF)%OJu9 zGg;qt+da8x!R%$1Z;BvJQ7G>6iJ9WP`zM994#h^!g&c66c=rD>@=igT1Z}ry+qP|+ z)3)umZQGhQrfu7{F>TwnJ?*!3;`{dA5$BI{arRBsT2)W1$ct4^t&EIJ`1yHgJRr~S zb@bspk{1pL6koUk2t1&$SU%$Ka6x}1F|qvrk&?uStP&7f@E`;E4VOg-Y1Mg@`mQpq};I2gWLP`)|A|QRlK;?JXdSb-?_dYFD`oH$+f7FrxM?^YDe=o2vM_18% zc{G?aFY=j);$%<4}K zvjf*im@IVXKvqK7CQ>UYB5Rk!qSb-CaI*4jruV zixk>KLqsecp~c`?Oq{Z-Sf|XrfP>Oq^*9D2)O3SZS$Da0DWgFcM}ZdGB_$DZ0^uPW z-&@;yII0BRMZwj@Kzb0nxAf1}FUv=Ozy9*vu1y#qu?`i9v+3W&sP0Y z5&$$bJIjVsQ!HH!0`)m`;udTl`De z#*V1ZY4c`yGL6}7a?C5t$~Y8|6~#81#l%-cEa0<}o*FbER$4Bqjw#4Hb(!>}q*U$? zxNGzHJ@>&Ze)gHKk<8^Y8xeMltHL=n+G3MBY?cndG|a0kdRDBabl zbMYe{=v5J7UXd}pPTK>$rqR|HY*W$ILLSU+Ij+ z8b6ou+_&%oGWQ_}7PW{Kh08l$8O{8yt~24Lnr9o- zxO_!ts&V#`jo+tj#hC4c+ZalzanE)!vzH2PMVxh~iuTu|Vs2{0(Fc9i^!nO+Pz7NL z5al7lm$;ZR%w6IgHeGyY^ z#o|12iu@Y*r>4a5`3;aF$+G=QELzciqqi|3FL7{1@*}v3MGkq_7*nKbD)I2xZnB{S z#p&sm(iE;qrYUS>(IMa$KoAm_2c3*B%ls#=jYIN%vs%#Y*U<-zmy@>hGco?F+T8bkNUK z<}2mbbtoJwGZ;3l2U|9xE!_+QK7w{N6Xy?`?p&mDt6Oc_^(=}dBGRReUJ5KnKL)+N z)z|2zS6cy~SCUS97|6P_tCFjG%We6a-H5iG(B*BFu~i8~4uy2vOIRvoX-vQ##ah=D zq^69un%2wFiZ7Z;m}Sv+W@Qoc5WT7kKyEgEg!XoZ7@yh33)1F#es@Cy0+l6(+4_1% z{07;ZNnfr=!5VkyS6|K&LuRCB(BDY{lIN>VpMppm6FVp!(FOGZs0nG zFS)+`QSxp(N2PTL4=L^#yGh_9{%>gL2?AHzIJCUWpBSg1So;5NbpIKBczAfY>-${% z{GosRIl?wP{}KARtllFG{2p=43w&6)Gx!x4@cH>xy+_c`Blvy!xL0tbk1!9Nnvn5x z`j+98l>$Bo8&$p()K83G@$)0Szoe^n_~u~OQYwp**C3iCYBOTJ;^%#wgPhy1M-U(Q zYU1Z7j!=K&W07+IL}Lk=D&M%_Jji1#>VHx`llwYh#Q`+Kd=V1qC^_z>2%EpDa{Ljq zq+-Ymbdme>1kqa)YW7|l>yT6Hwv{7YlR`Bz?xey}{HzUC12w6WhANnqXj0>KS5l-S z+aY-?*vmsC^yBAOV_2CxdsDak+-!j-*eFSBgIwc!pHI_Qb-v)xgTDNg5G$(P)kwiU zYU?=)<1XL+{UC!8{+Cr!maWf z!X9G{oR|zf0EP+wHjH9nfj)RWO5hq6KJ{iZP(|!@m>1ffY(eOct4f1@5cM2 zA7xHC+KPYKlns%?1l`vk#cUj=L-NpGpZ^_bYX9(gfPB86L{TGcCAk=48Q6|o?>{Lp zFSKxw&+k5T`L6r*S`g3M$D+uyz#yW-E_&EwN1ofdKr2>bTNq$Pao2%U-Gu)1R_Tna z8YEluMNYN3*Zupn5{nRf#xOPKauJ@sf4Pi(q^FGq83%pq!li~{E z{ixfFA=TzV57x~L9jOdC)Y^@Wz0<$6e;Zul7#c5j-1OV)=<=#(^z$AWY~8wL*IAQO zWzpKjg>eh1iVvusm8g;_+Kc0z5xJ z+>J{$U*&>Ad+=WmT?K&1+E|=A)eJyTY9`V|E@DYOoy?m`k%&nx)lovO;8Tb^&Z8vc z-mO63X7!8@6`KYPPG1lj-1sz8y_%`1)WQzMY^bka&(){|;y&=aizKbo~T`Fl~ zGL%+OnOGy#W1PpkQ{dPu|F7Wc*!l}yQB-04KQrdrSMZ(k9oE~-8YJ#bn_Vj0Y=RT* z-u@ecD&sh(#5e?KyE)Uq`?5mbgjpy3?BdpZNuH_oT;UdF*E;eEX_T8C-GdvJ=IrK* zP6-(pZdBrl`IH9&V~+A>?`q^XsC>8iD`XrunI}Dg=hsHYvqe%em|?{J08C2Fmi@P&*V?%|F=BCIX>UtUZ&F_s42!HDjiy$Umu@ z3)3<_nw9T;RYHD(^x9unT6Z$469c=?*|BK|biEQy(*e?!6TIa4GDhTCRe>li;%ZOiZizdI~tW$SBY>>I6Nd&h3~+NQ+F_R`o*n2NVhE zRb50&EFhjCgGb}onHm7F%umY`U7#(RbcCz3&Ck8dokn7?H{w)BmJu|@-a0Zxz7%Ix zP3pfw_{SJ5SEi@`DBPhAnLBC0-vxid&7reLDdoHr3yROAoNurR0t;gT7+Y}9#8alt zlFmI1Y9)dBoL%LII=wmIG??h_S23Nizgs*N1D)pd3K|&k;Eq}hjU8?wWW0!2z{l+< zTu}#O;!h{RiWhHH++y-B6=Y9><|FKeCwKoQ2dSy6bME1gAYj&fw|$S3XqFzD2SYBP z_~e(j40H`=OYG(OHV;jQ5svJ^&LN{>1yib9xYu(a{3 z{^YktY@2a{@+2OD!i8|(mBOM@@X=#iqJh!Fl0_(7m;p<|3A= zxR?y5v}g@t(AkDS9(>HNt;!c^>;x733UtoFjnm|E zi+E#cOG!3l6+E$CUimt1q%mYQ#f^S3BQ5Art^64DmW$G&S;j=R%U=e@E##N(rlZmr zN~tq$n6OTFv;(4-1~U+Cz_PjeaEc%2|Gke-egHvK3+)_6cf@V#V@L6tQ#o~*Y`s%^ z%j{B5*T|Grjn~u{RskziQ3GM&ggj{76Kcd?uE#sQYlIM)bLY$GH9J?m7km0fsRLqe z+~pHJeT#*J+LY9?btb5@Hi_;5Gca10zuYVNk!__-2-5t7dWhANz+!rE`yd3 zseLiEoY5v#%Ek88@WUF#yL+wUD%-G;JrcO=vb8Y&rcJ0szmd!wVMCB0_)yM;3&7pM z{&NjN3KfR_Cq%?H2{XKfUCu&OLu0v#JUZ8LD^SgRf3Gln#s#P0xWpFQ>IAadBXv<7qL?q?6TfUb;611@J8Sf@++Wo#+!p7~;N}2e# z%hxdv2jCbI(RT(?tyh3h^ZgN^Ml;@U;aqk-jM^dk1t`X~K(3va(Y`eUCZ7Vw5ROGzlOzg|qgv zl*<3+!V@lqRg!=?+eAqQpEd47wUKjVfUiIo(>GN2 ziN#u`ffNS94sg?)C0KlOhJoHx_ZC|KY10B;kn54Rkob=UB=52(JVy4sfs2}3 zzuAS_Lk%@H!R@@;9P4&r5I={pr`4&KwCN)MI*glVvFW3%LP#-9_-lUN`~Xlt8`-N^ z(q5~(@+o$5<)z`I&#oe?1p9^+Q5MM*)pk+A`Crm~;zl%rf_0)t%=paU{C32ylXWs4 zW{ZvFnl3`XSKlA4ytV4iyAP(#Xm>CV{D{c5;2#atJ`PTK+pD_17NXVJu+70MSgi0B z+Yx|a5nUAnKSyBfhBVT)%95)tOBnfhDaw`dI@NIas-+vDOgBY--i^^n`;yQKDtK&T zyFv`d{e$#ZpR27qBH3*4uyJ2Cj+~Zn=jV4dObTlr(-VAas(Nu#wB*)1W*Ay4z+8{_ z@4SB&%qZXBJdlF}&aK{egp9iALhR}_MuN=#eSxdjH1R`KWGeTC6pMCMBz!e}9B5oo zXWo3u^oQVcvXkqaZR-%mj_;4IL$h=Y;nQ=+tcOIEd%iw0b&k78|DE-TO0;-q>JO~_ zQHbc)-IbRi1`y;%ko-&4svD`Wk$f#W!MYcX@7^M@)9a#RB+J3Zp-n4hmCpr0SNFIl z+c^(?9ynOaUk0F|AAF0==E-R;f9N!7i6)~<*i?egx!_yndtKm+b!KET2+Q2D@5p91 z<{&oZR13=6PC7{*a__^4d6@8i7!JkY_EN4Lx+vpTdfLkiCH`c>H^U`930U&kv7Yzwq$%dwhU(O)U-Q?kJsNrww&M{y)KewVl`f-4U0810fYF28 z!da~`FxTU-h}qFB-C}eH@da;Wf&*dA~@_}n-d_TPo|Wsq3_lo#uuYlm?ny19TKV;G&{-b z@FvQ5hM?!CYP7^tr*P-J;B-RgyyCu-1VBI2K)~gfiE?I@6qp_cx%T3@V*^H0baPJ*gu0mfd^r@NuI{8$G~iQ>cSXqu_1;9LAUqgG`565X2CQENY= z0`^(Hb6XR7N0-DpLjpbH8BydwMJLjE^mM}oyOK|{9y$t~)Z6!`{+}})!stPrRT6|? z6_l7@qZDEioKn#Lg)tzC34~eT{zGiQA&Na--@bxX=-H_^r5$v0xb(K%Y6p_24psq6 z1r9F_H_aJC499I6H;ow~%3QSvnzzpO%-q+vEL&bXO02s<&VcN7{Pl>NQ>i1Oq_(!H zcH@@`UD=l2q{ho1y}d)}1+_DI!rQC6zbz=X#Ymzy7l+8PV&N@ zD9tBf+Q$C(m)y5SAp$Oi4z7$2H_~J0seq2#gV-5SXg@M zP2F4Gl-;e@Us9UmOgx>0RU5d!Z9K3xSFSZdxs-IeL1U_eJqyim=8;|5#EX{?MWjrs zQEi-BLxqmP4Vop~f=VFw2>Gc)hmtuVg~k=IDUK&27qcn46{IU zP~D$2ZakAAPVnDN)4rPAPVo{?Q@@(rOmGuVQ@-L4>vx_Wa0m9h zxhK8PI(`kC$F1oc81!^c;vuVtk_}sT@HxdZJe+3>_;#)|&*2Ni=Y~0KAA&LgVLGhV z>>$dp%vUvC#MpG`htyzHOsbH28Rqq@ROi#ZHR3b~y$oA`du zHf(Xd9rj`vGb4ralK!i@po7j9{&p(itr1B6kJGn97Gffh3$8O2(<9H5h96M z0~89|+e#?H;GK{7?xR(viLu2{Ts4^&9huwC4h&mn*R2P__SXI3HCL~nCs&VEd-u}A z6fXwZt<&|{x8(6oY(tIGa^^d)o8u6L4paw`*qt2*I^$RJes_Iio(-;(8Q86j-S*IZ zQSY5V>|a6x4@8n{hf&3UAK+PA7)cI$`(>R+zFpf-^KsZQWfe6g_Is!c7iG{#Qrkae ztq8daPEZMLzhYPwoS>ZOI>8m3eknKsH4*s^p6W5&27?rN^bjybsB=hvADR+^jEoh$ z;b0Jl9GV(~jEwn!M8}IfG|qb#ylL-K7rbRa7rbSJ8G1gjo`*hm=+X;Q=fs!QgSia` z#lg}4R^*9K)EA@9iH|ZBqbAtjx&p~{DnLha8w~nSTS545TNxq`{SP6*Ie~IwU!7w`Dp=g}2p_KP$A9*l3wm@i{pdoL8pgxDcw!@L; zXATzJU4Q5C%%fX^6i_1t=kffNDEyC9P#zEcf1i(mh9V3qi~b*DfQBN3gNj1sjggu) zfd1DhLc&S}=TS=#=TZM1#w;9>mQqfT&|!>I<`LK&OyEQ`ZLT#j<$;!D#0`%dz5s=y zKhL!whSlsXU*0WFLg5M#RdQp!fg05xUh*YbelZ0hEQLSw6SwGF?k29e5i@}T61nWQ zxd+te`VSt;PPK-K%97P)=VcGOGhL&q8ODbqXNkPJJ^u9;?v#spnKRymyJ~8qt@W|8 zK|IwItR5_kr*1{hx=VLfF%G*;!P^-;k%BVT`NHbdUZt4)U19bn6lWb)(i2$|xOHnG zF0SgPZWcr6>MhjmA9l$_mlt4c|CladdT(rDImT&{0@l!39OT&Ds(5=}#meyN{ zn^Ut9JH3#~wcDwEc!{J)lM3xeER!(l zX7k5yJ8=VXqnxh?`#eCvpI0Vbbc3R@hsp!mS3bygCRX%wD1?S4u;-=wKZXAJ_%8e2 z`W^T_r@4*VubW+0HD}#JNSyi2HPF`S@Qm{Ut04=#GbV&4r0Es{CZ2lP zl(UVkl}?bCI@ww3<%5)&0=qU~=wE;p>k94%YkWL)dgWT5a&UEe!~)%CunJ8W^=fUP za4Xn~O?;Jxxl5iv*2+Nj&(Gn?$$PezW4)4L{!_6v1ZsDt@uE+bf*xO*MG+U6OJv0| z?lxFz4h03Jgs5AbRXVem;=F>1#z-i-_YoRxy%ua%S!xH9oH+bh+$jaQHVUg?r235o zZuUMyg+}i~$<-hUkW$?w&XugS*nJ6eyE#yoz1`mMy)&g$r{V9QNML7QpRkPNN+v=9 zNemohN0RI4MZd3yA+t#N%jh;txPqF95m7SOVzG>)rMa=h zsg=Mhj(;&>&|}4zs-BLnhiXBFlbTZv2dsPd&LxS9P}_qGLwgHivoPhFx@VWO8wF zDgvIGM@o}80cDM}H0?;@;U>K7ZQLzRf9Z%_3DWAgBtrbt3m;JZhxxvE?QV8Aw_u5* zHKv*w*q}SO6FrE11YVTOmdc~ffpy|b5(gD&i{-u zQhbY7%7d5<{40nIsE9%@Lq!Xu{JqoLBe-ybJDX+XXpN@(y!^VIl4o#L@Z9ddWvQUD zld8_#h~gOoom0-`+V9M~+p-8;h4A~U*i^KA!ot&{pR_&4E0Tf?9_fN|0t!;(he9Xf zwJxK3lzBTs_$VKNWZlGpk2-1|7Ke4?L?xOSa}VF62)@^+O~PY|hL)N(?@ zcJ3eD2JAw-yqcs2qV=)Ec4^!F#b1g94cJAEH$c>obV7sY~sOA9`=*;GmVa%wYr== z`r^4A`Pih0A@^J;2XHIvvU@p^%)f8UA|?;}`@s`7B;RAm)nK)w6|O)OrL7J`Fo+yd zI5E1u>aLJ&J8hDx<^bO;v(}=UJMKGAO(6jcX+J_b0 z(6@|LVJ#>q+^v-^=n!tL4@*W}T7*Z>K+b*_8-)v;N)I#H<1oEG$dMW~|F!T(aR8Bs z!ScFJ`^9zWD*2A+Y+EQuxnoY`L3;@rvU7Zy+tq2Thr+WC0Yl&g3I>hBvmW}vqN7^U zWqk^H%2TpC-$uD!#5IfO>#*e2-1X10=lwq|0H&ZIv4s(L&|D_E3%2ut(BJc&?>Dm) zL#(tmE}MYu5oi6*GA97fW%yIZcN8of;4A-hzf43)CjJDGE*7J^azFSxYxmBpW(c@H z+T-=^V;b-0`QG(H5gt>W$H55t#h$iw$VqGA5)hnsH|tJ(x#yucS}+C%h%5Gi^f=&R z=JoCWy#41MqqIJW_*f^H0h;p0oMb-e>1`5{a*n2#uJQmssJc@EBC;6QZ#X2n z4rF0SXfAj=4;W!cC7M$`B*4Xvd@aH-F2!&y4d@gFIgJ zroKSpkt+|qFRFb~&1j+I;H0G_B#*hx{j|ED5B@(Oxdu7KjuF}b9OBo1cLEPLjBoDy zChzZ$Vts6QQ`X>KR{K3+57Te(1zGSwPcfKygrnGiJf3OzkXayCPrdUSv}^hx(vQ(B zJi&JX>A*odNJRg0ssO<^+8m4$edqS=xnajz#GLKeQTS-lcKn}Lfy z3W0Ok-@@wEprL5X`HQM35?Nf<5oGH`GY#4KYBJ37e<%l)%f52{iKk_PJNvvvXp|ko zv@&edGDqmz9fjqWTk1u;TzfevY$euO=FybSrDZd6k*Ac3B@R24-6UE6`m242BVdqK zXwSqImI)1_TS<}c-Tu#sm`C~f|G$gbmiarDrcnsg?Rryf^v_%{ZGs>JZ96?dMz;Ru z$iWK7E+mEpEOxgC;M3;~lSh8h z8Ck{P6^TIo-q-tQEfZ@gZ3vpGvM8LabSsFgw6y`ceyd~dQcuq>e|8_&7s>`4Bi6qr zx*!aa%^7#W(ag}SnL|7ylAZ%gv6o0&{vN85;?5-5?jlj$9o2n=7}1+isbv{zB?mfN z{WeK`2MnH!@$!My`pTTtI;T<0uSptXs}y4DwyR4g%=-Uzzth{N?r^}ZA9=~k?pL{lvGuf!GAuqfyw5APh9K#-eEd7) ztbo7DJaYiL0pjjsB;M55iJg(!{4O-m{CaeO5%2QYBEs(Gxz0;;ekbPzi@TuOrixt^ zrB31;udns{_c;K`dqA^F)vMN?pHM8f?h32JlVgg5n`$Jb^1#?ljl92+FbnhSuqLTj zwqa4ginSbFzXjrf?y7s^cBehBmrlmua&{H88rX!EbYkO1C7|M&<8rsxUfr$|u=lVM zOPFJRt@wx0IMqD2Ixegq#J>6Z8ppr*26D&Px_Rr+f71Eom{v>84vog{-ee1ZZ70g}!a3$}LWz zF$-Ig)FnyHkP5ZgN$!l}e}QB?~v1MrP2rz@D;3;4S) zi89r`QF}G1sw_zq#$uoQRT=3U#5F{IG!pGPfnv(9RX*sSsTy&e=VdRKC2D%!IZC=&T#kaLb#bsddj1wwu*Aj z_4kl>99zD!t=@^vF(Xr*zeh&D^F`LAQvHCQEH=)HKwsO9&kviS`uex3o9U_Ch1pLl zQ040;p1bSHz*VqiZVaJ7?S+Q(AaeY;B9q8Zy%@0xcY4+lkp8I(RvMjT&K_9cSF@cYz?Hq^MId8}6 zNLwc)t++aH3&Wn1GG+-^qaVwER-rW;U2}1T*Tvvr^LtdH%iF}a{Y_y^N?wn1G~J=R zUD9P~ea&WdnQwkm`MPwwOTdRpf_tf4bl$|`*y=`cEm#$ITtA7hI0uVkY-7IbBJZPy z#G;!-5ls|3w7A@ik20zResSZk&`AwLYLwC9Q?{VvgWV@ZQhzq=WA*BD^6a|;yn8SQ zVYPj;s8!QXH&3=O@4%)ud&ScSoI>Mj#quhh<{YNrYF%t6RAM*~R%Ri`N-}%#LiM0> z&9PO;wy3)zH>(=jP@Ch*4rAky;hHl4B}kk*n#g_!dKBoHre7B~3T$ z{=hC3dtd~2HUX3zlSrR#IJ6v=8Fi9mbb z&II12eAk5Mo9Z4dpi*LQX*ILzWC87@8^l`7G}UR2u{4BsB-r8 zLNU8aJO3UsD7DU^FB(g|MAxh$2})cdSVA1A>I7MY)Sl`tHJf2}HR~m7?c5ScAThc@ zw6kvvpW%=AVPU!R@tic%QHLcG>Y6fB7+x+zcbnQ=);8|kJ^Q|&)^OcCr`VhU!)9xj48=y&XS?BSB^?Ow56h&5_2aXrI(@{nt_aIXI8Z%0n=2x8_WC)RHr1( zC{)Aso7B0SU?VRBW`j~Rl*IkT*j`q9X$JVHy?hcrD$}Rq=F$8-1T{UEGHEuzSj_AR z`Y&q}++wZEEed-E0JADOB$e!}yylcgBu}Pso`pD!>$Vn?=ACHbtGaYh3%hvS!j#X> z$u)e;tr_V^JV+RBiGga2koO*QrG_ z(*4IsYtxftv$S(Z?nijgWHB*neWt)3`^!`-Z~uaiNXkE4%ureCj1OmXQL$?-w@r?nwc%|kvZZ{c@ z^OrL|6uDCS#V;NWZQ20bKw%(CNtCU?WF{(6e9PhNaa79l>QWpb=y5!}WdvmUsivDO zwPquS4qR_7BZk|raI47=UOH~3wlpKj&gy+J>KG(hyNQg(P&BRN1g5^9OFWZS(@CaL zMF1lV-o~$pxsq|fBRuk5I*4Up#e01Z-{%LSQ8OV3e9L6h@_6pd$NOfZh9`G%szaRC zzsr$02#d0T>3rD+pEvva{eAw=YeGZ>pI=mF@%6iFfL*UtJ!!YW_YNakm1|On@&J*R zf2mK^($Wm{tagoil5XfZoyikobAB5OA4P8t3A_ZMvK z@Rk~JDsAVn$`n3wM43U?(oBN3K_4ZoIp~;MDsi^bN^*}sw956J4{O$zT(MlzbME^- zp5=Vx_Qn<-$V0IQvP=6KpGG|sCx?Enjzk)Jf=$(0_!6Yu!sV2_2w>(M<(oi7{d(P? zyez}GP|LDNl~^q%j1S`VVXmrcSx7SF;zf*Z3qBiQ+%PR z^ys>T{@IDJsQWtu6SjKv4=|)slDp=$j|)X!G73f`D(BP$VZ@{m0j?37khT<_PJ5n9 ze9|ir+X*J&BDNZ~FpOdd^PX!j%F1uPu%+`;NONM5BK1Uj(-oWN>)3zdo7nefkYLRI zI(Enqvb2OKiPe+x9YCj2N`xIxu~nmuCYO?XmU4q`YT3}ulLJWwj!-#M{o4HlzXJl* z%nxofUdlXisGh66ok;}1_kVY92U}Uo>&)#wc<&PmsDLb}BNqJNxDwVY7n(j0`gVF5 z*`R)npPQXkRO6I5&U-;NL#6lzQfxSJ4?2o46%rs!eQ7IWqtlMLLcW{i z9RFC?JkKhB;?GBT4dS?`FnHB^bVr7*A<-cm4jY<#RANk14Bn%Q;(_l91?2eyvQZGZ z*s=hqIX$jF_3Y`cLC=D3vZkN8g7F!dvOl+|(3PxEZQ5T3W!g`Vrn?D8?p=Gb$g$dK z2u?+~|GS5qwF6)R34hbwCbO5F#M)V7@Ui&h_H7}a^0#d(bk~J20CWiSy4Zw04sv@sT!jWv`2ElXjiPTy!yt$?kMzAz8MhrQ#HCl#4O=P^}hEPAJD zG8zHGl!}YB6U}6YH8bBXayTJM;FCc*`BYpZ$z)oZygHimXs7lKk-VNHI>1!L$sEUk z=ay6M*8hA#Lr+7i>2Rb%rKDds0p+)_mo;Mj%mhE^^(h#fZBBCgbsFc{vCF&WC4CBT zbi);8mgpfV9r2DpfWDyJ@SqL%dYB3ARaaEnZQ*ur0PS zr^e^&bg9pnYxu(vP7!;9^MgQ}BWVA%%CXb!2Vq@iC;pgc*{*z<`l zw3rEp=OZxO;(X!{!m@svFG@fhma~S9oyORsGYz8dGt z6XVlz@(OtSxU^aBPp}=JSER?(OoU~>!z%l!iIs_J=A@vU{ldBI+(VGE(LJ#EJKC6o zP8sD;flMCOFDmW6M0PY?6yWpk;y8yzKPQo&R*^&gEXVIJgypxX-Uix$3uDRRS!`w7 z-!T$>HUsc=i6TYCC9`s!18?7Z2C%S{yNS{Zd^4#&YvRkiW98uH21r>;ti8GigEwO93| zBVGSCa++R6PTOGnSaVrTZ`WFHOoSjt>)i7kXRmQ1x*fG+rIr>2?h7}!s(E~f+v{S? zFpwCQP(yL6Q1FhYjSM<%G*asDr9`sP~Xo_yC#SxMGl z*l7`s7zy->lBX(dQVCvt znC{V6zfGnC$nD{jT5u7!;a)DDY73Nk1g1+3Y_Pd>oH{|JprqLz|$RCVWZuIccb3?!$zxNqH zv$)Ez3~!-vNBDUvsPpTO40_*-qW0$rjly+{;bGp4PmRNRYufvYykgyb80uL_uMD*p zao1H^3ZtvcMIBQwr%bp(5Jb`p#|oCM+F)lZb3J4%#>M#8Xb#h%=VN7$qAE67#!tH-z8eWOGl zC%ht#AzHHX+TTR!HAQ;jFif2wa+1ZfL>nh&Y3Koe;Hhl6x`y}djasyYrUiVTRNQaQ zi}`>Rae`IF@t6OF9-9kN?oTQ=vA6%FgC0*Rg#PONWYvocQkZSU@iqm7{_0Z1EB~!} z-s;cbaDGzx3inr^kjg1On>_FGM%B|YGCB);hMh2HK7^itG>?~ro-fsNa%Nh6Tzb!f zA`d@2pn10lz~(B?TSfaZv^++x`LKe-F-71-U^ZVB%8?OD|FX#XM?4fGv=Ew4`)ra8U3r>u{c-yC6jL_y++FrYz)EXTHOD<uf?tA&R}aZ1$|su(#g__gG%GV%O@~4IA8%4_r5UQB zpZ1F5!;5gb89Z`M?L1C;1WYE)M+NV+KH-LKOYLzwm|xHz;fVL`D^N?-GrqXl)Avxa zwl8{A*wcxI$xXIcA=CFVf^kA@uMB7BrOEyYIxg~kKdet5Cy@Z&zOm~Mm*6?)u71>2 zenz?NxwQ;h-e2MI58vM-)_o(sr>DOXlpa_=qo7|Y6*(g0?@5U!1K+GNgPQ%qJHH2E z`vw6*#&PlY4c`OTm9wZ1O*LLi3|`4Kp2=UoDL`Q<&0905)<*3Z!zZoau5J~t&vTdt0U2AfjQ_12%7ZZD!&tFDq#zJpRA^C$*sM)iQ8VK!I0muRK5)qU@INZBrV^8WnH z0kFrO%JmG=(`vF9MH8D6+iK!;NsLrjUAYO(__wCo9|3t^p%&{580u%)r@F+sjVxJ| zPksE){&|ibkct=rCN(gR_v`V!r{r4tx+p_c!&QnBZ1*0)NJr!wFd-KP*<`kp-w@p* zqR^V|%8!k7Udf~#c{MRO=&!jJDig%PQYtY4j%t!_tqz>DC%HV$tHgCMl<;NNARJBd zlBiVYMY608f3f^<01QoL>g|Z+^i>;Xm5^tGeTW_ZIHU9mwben!!$!@c`TT~=s*%gc zjUjK5NQ}?NUL)*0LCWSBCqxcD^9@vFcWqd=F05d`L0m4YRl&&aSGKPD{aqgn>-%x{ zTa~~jBx$T6e|&AfjLY*ZhQCsBO}S~kMNl_o!VLEj|8OQmi<|-l zVCCYbFnGfBQqDxu!0(T$J?ra1Q*^jd9%C{f-U$0Z{_d(2wGr&*03sRtLZzbURh>(5 z0h6mP=KvR_yqSZa(H+|s27Gc8VC1v|;m%5kD2<`|fH+JUXeF!cYb?VJ!e4U;F14P{ zREG)+dV>T+Tg{kdb!^+FLOs>-GV80z3JXa|&fH{Xb1|;loa|!r;n*u*EDYgq`x8L)VHeudQpmc8nceVt6^pmdA6)tUtKvy8jA6JAp)4?s+Ui zft=;rJrYdye~SiHNT{xhEn-m}u6qbe01@x=LD(o_znIdaAg>3&*wR@<(6HAd*9_%K z_cn(X=KS~kZB2P(f=%wOag9YT9FFK!WrcZ2R_$ic>(+-H>st1j+g;M99(49pR+A+Y z7k3s!E@O4mTzhsE#wOz!GY!!&q#Od_U0%4P*PTy8F(&A|;q^L7MM(Bdu1Gke%qM^G zx|6pum=!HY_39c@+_?Ee=a(WL)j3~ngaKbxRyqKGK!CqVp)?5U49om!3u!)&ho-8< z!=7pu&Qi{=c94(KpN6VSuC?Q0+`u(-9o9Xbz^1Ntrfx~#LgE|fd2``2RZ3a?8Xjp)&*wV#?C6^ zX~OMlZn_rIz9bw>O**?5FDlo6(m7cjAnCChwCX;RnU!ptZid6%<0BH?B|weemN>&j zDsHJ~B>WzImlrh9KlTbpkG9Px=1xCCcya#i_d>|X-GgI_?sfS%@5X#m=D>Ny3ZFxZLEbUR`x|+z8&j3xeR8r< zsj+B#I^G;h-P9Lmy|#iT{$m~J@RNT33nJ1jE)WF!y)Ku(+!H|dcZtojp7g*qlo{6= zXZSZPyW)JJrh@ZEi4Q$K&Z6QH^etvj|^YI0= z3$b=#k}SUf=TXTAOuoOXKFQWNCj_*yd|kK8`xE_boMU|XS1zAkcPituLr(Ym2QU-y z>tb)s#H04fqOun~+z@@+NQOgf7%s1?(>&;8xAP8SyxPFH_m(j+B*NL*T<4{gA5H@x zRFWX0SFTM5w5|EyuLP=iv0_@#y1FxiwQ5RgbEuHADnrWAc!`WABqkiaHx}s-G~I%{ z9+k6p<{4q?HH8T&tv&Fuf5im+@}zR6Fw(KL9QJWh*;%NK9?tfs#c%j~i$c}pEcv8- zd0Rbq6n#E#_T1Pdl6hZNejktKPJUyAYVkZkQ=eP4V939zAnc|(ll}Ej+(;lwIukEv zO||M5%9vg=&wO}zg~I3-V9V5C=cDB-4o5?hPs6WR9nE34qRF9dV29k@WpbR$a(KT# zRNN92Ngrg8H+0p#1^SD!cPJ%Km)pTnq`i9=Ur=TuT~3T}a4*!33#0#6oxG<&lZrIV zI!uT>DfB2h`7w5jcM|!jp}G&_Rw`%BO-C8-f-F!i{{=!Fy*h z;7$*3;VahG$oUuRR`J0s=(21>yid87t4kEi+2c3*Mdxz}A_47eYrBw8qN`CmPv`_0 zC|w&;5~w0dQyHCbu=i+79IRvI62~l>ag^Tk62G`^vqZK_jMbEu3x;MB*LY|HH07^=9ID0pD+JPHo$^ zZQJ%$+t$?f)KlBG&8xOGwY!u3-}_1KJV*EWFIi`6<@*NKrxUvnq-t?oHcM^>M!{KR zyTfdtwz){(_#$}&RrBnD<6chzuL*265U&(|%aO#wLX6j>Ycl|-Y((-yxxT$Soq?{S z>6QkSqt~#cK2`oY#fQ6*ouw9RMSrw4lLUqjADWyCuCne}4oLuF(mhs~_nGwC68p$So zB88io$75r-ySW89mgVo2*xaXcAKY=Pw=lQ2xYEswr;gZFl?|3uI)rj%-SN{i!CM$76HvYsev9uxy zQiM+3^hTdsB7uu;Sd1rqE-h!C?qy{daw4{+usT8~z$$~)2!PFy(%YPU!Es@XR|(LL z4y=dE_XH0~zU_P^asSL6Lmnm^jtwPRIBQAbqmknFSqI3xzuCMgs&BriUNHF!#$Dw- zXr)SfZq*Z0s{9cE-;C!`|95_=>ir4>!KQIIc5Ri@zwB(M>s;7tYmV0w{0vcD$7q!0 zZ%VGL92Ev=P2{m+lCYd?iwU`JI|;kWjYgu9+L>o4OiSirt#E(1sWjF6VH091F|?u! z7~bf6w#kRKo<_nS8X;fA^{aFrg41MGA-53jqc|Acd%Skh?kBOYYE#V&&FX1jBv@R= z`awpb;GU=%MU#7!?k|ITF|O*zY=Ury)4F834$-iL^491Wyr74*Kf&uX4YTbNYay5a z+u8t+v5{$y)z{;PLGkuGH(M}&U?;@qO1H@0Qfp0*_kN-}#7R5KKJi~J8cDQmth=`G z4i1!qwg#Mii|Y;RoM!4#ZHv_)P?q4sVA0p=l6mW1ZzNl0`hF9fZA!U>&9{5b->C_Z zSw^w)FC_$F73;mAeoIEfEo^2qolll+#!^b#Mif!00cez;M*ED2buERUm}Lpo5r9~i zJUD|=O;4&KXEiepaj5_NJoy&I4gVQVI{lmC)lE!-hRZjPJ%ss^Vj}+qTV-~Y4-O>3 z3ZinIL9XO$YX%*#!`$N}he$3hfmPj@DGec2t^kqJdWHU&SzRWv-fEq0*A zLFwf*nfW_=od}J3p|6akCA!PgQU0XyCIJ>OZ8VqJ_**>&BYbOFD~`SLq=c`(cY|l1 z*>PC>j^%DvkYocw#9Ez9%{|Hbhf1vsRI%PStDWi54-irLMY6LZ|J*S}dT5yo-l(KA z4|gA5j+Ju7cy=>o;6&&%yXVLQ;~w&h4RR@*4I56UvEMzWeHR(~$Rb}()y+Blgwxe< zs!_^7E(d>^5=lC_gc(#mXsV~o$6paw(`2|vAc^bmQeMAhE0MyYdlyh48+_E022og3 zn-u;ihW~AJ#-B92*+%&Ax(k`)E-!|mXcUG2A=degikr00d0yOmNW$jubI@BRetE4& zxX?whH}vrdG{s9K0A*N-Fw27eZB|k7dYN2eYvNW#(jBT(XNxq4{DQZ`*SHs>9#Nss zN)qx`D1#ht`gkr|iDkcNqO*)Xr#qqYUdwM~Vj-M|Gdng7ZUrOzZY)ysgT#O2TGlAb z!l+D1AKV1rQ@;Rv2`3q_|6$!J^bxL&qjG_=wr^W}HZ_VM=BjJ3_V4k?oT=v~=Q8&D z^3U};Hx^VMnl7MSm1aHM#?!tb{XQaBQtogX-Ke76qiJp$W-+><<1hbTSayR#e`_SS z#=GL6@P6E1Hh_jF^!KG8cd}_!qN_5XQKg%`gGVn zu-1mtV0{46G;)nP?{3p^sU)q##%Hbwm0GD+l zXToFLB2SF;u5})({HZcNE2v;Um^9%UoOA?zuux7g$(u9`{w%~(gtA%h?Rr=8F(H_@ z8w%%Ku@VJJl)I>IDXnz7^<`ixtZ`pUAn{qpZVDrgaz>p@URS)-k&%|(=waOt)_Np# zRwshDRz7_bDowFCJgr}+B}3lC2-qtF$m-cvb0MB9klM7|Yd{k}D89JlE6({>7*uXC z`#1;iZUr}7L6qe1m-FnL?BYM2AT)YB7pa`Mj(^q-P`7j#CH8oK-~Qk*w6T>|aEkH_ zS(pC%LYhxM(6%@Y=ph-Sh%uZ8l8KHcZaGvW?O5u@v!{6wEfryJRXbOZyE6;+?yPL}U9?irN>?YLAP>`IzBlpj&!{SxLIv1 zN&n{y?Oe#~HWX_s$uU#aZKReo{h8$$&v89#V?%!K-~8&Xfp0JG>)L%? zHf`haB@7)CMvr1ArFU$m`%%=}sBAvbRMblAP964KhKXcwes~b2(@eqeqBjwmRpFl+ z-ThHi6I_p=P8$gkcPa_dS6riH43LWIps(Ax@5ci329CB%Y6&uS+X|awss>Y2i&YqH z#G{0p()dv%;q!!72>zc>%LOcg{Pe45n|eT#Z0!knOTkSZy_a$9$!UydN65JDj!AD@ zFhP1eO0~tdrZva+;+1`7gJ#1QcizAQdX9>ePvnO-ecokJv$mTZ13SNcE(VYPvz)F5 zyzZ7?;_i5w50xQToc+6h-QWF-OU8k8wVC zbrj3@SX$-RZD&e*uKKLixq`0`qJNnvz(gTIaQ;eOsR#)E75z*+T@A@8SzMi#uyCS{ z^_QJBUPpMV*i<6H&$cAIv5q3mYTbOZu&}jd#l2s=4O@t!s?RA)+aCcPX&PN7b?Xds z41#OELpnONwZ!m?Mad(GG}j5d*{7v+`5t?f=_bg&^9hJGc8r=)8IETJ9$b~XpomSo zBpz?Z8VpHJSF}XfS}(~t&$!=s!N3$3)``EE9U3b~J8_`32BP z`*%$`cd24u23%#|S)76YB5@01=%4lzSlx^dWFpw?yB7OQd_~_KY}p~M0vH{G(>hN% z9m93NUDVmdNs86j)?oP6~`0@hiARk)L(bPXyI0sOsAiV_xzppZkYvywZR^L~f zH2Jxp&oicu`3fB!oerTYd^+TwVFOuZi7#c{DNer7#YxFDpc=}p#BiF~B65(fG_DVV z%ZU^kd|-=&Mm`iJY|MJr*^dOOk$CZcFT1B|qotL0x`=bl)% z15&v$AA|FneBYWKgL?baAqf^>^5f&bx>z!wK=aQ=&3zhdIF=V>l`V;4D>Xi!Q9ene zBJQ~^n5ad8nWI{MZd%A#i$W?N`-$KdOJiMv=Bz?0f(PVo@wyf`hrUuYq}dY`|2ELL ziA{f;Y+ddBrL-724G-x?1Z`Jgbp1CGjvBVa+`7sq+HU)TdD{PK7dJGTf)`?5G#d5v zNLK%h|3+k7Q00X5MWg^V)T~*o_9mMGWhvtFHA3n&XS1pb$T0Z63SLP2jhVgcqlTLt zB8&IsBZzrzY`}|HMQ1o)`1M+2ijo0RR{dtHsIHI%f$;_CNYMGXFzSYO3##_ zCFVRci-ARu6zuuSts&MqTuWP~ICv^khkmv*Rm{6=Sb{cBW~Y-@3&)}>hAF?xLOFhn zp^tL1gwnxckFK{A59|D|FN#cLsgr5$Muk`yA#JqqNsdOlmrjTO)$XNP%6Bom+k&~a z(Vnzr-A^y zxl5wQ7wcf+JueVh8HIViKP#F@XiJPdK6$yEuBI$VQVg(=E8a z#IQw!p*JpW%q)CUK=E);uT zJlaA_3`epw{~PJs)AF4UkD=5~5~Kw|u8gJ*DiaRvTKGB}$Gef}o{vl$%wsRI?aIR# z(&~T0p!&eQF0IQ_VkiP3^efwOuuP{Ut?dT-0%&meO3#5`CK{;=j2!Vcz_ zY?md8zy;r4%@FQrtIdzhJQPRH|uYXKP$VfG*1)l+OPK=)nc7)m zY|Kul5KZ$H@@Rk#)$@W?(fTzhCx=q$+fgapYRHPXN6G)FEwep6<|L?i%YPQTUq~Gb z4XmTAkvG(zKJ-gsf2%~b$Dsp%*c`S)L)I~<0N9gO}eA};Tx|vpA~3ZnYMwMLrg%bHAs!V+mr`jSa2gXO(XFZEZ5nka9ygFJt!`0Sqy4U z{w85%>)%aIn);HClV*WwdO?fPNUBIr7odCi3mi$Hn&XCbA5GS(bNz!lixSRT202mkoef z*6p|4kyGeDf0kK_#`Xa(N2zvkK`v~Ho;iv#=g_})BTUQa0A|^LMhSAv&N(9_y$zV5 zBz5NiunGk7*<#`t!`RJR>!mdDHqI+hE=K`V2iRV__U$P#kr+x*KG8V8zds!sKGfcC zZBtrxy8-Q=_xDd9p?|C?c*01vF38px#;(M7MHl8G-E3j}4U zl5k8X2R<5;7Dz9;I+FXr4iz57y55GNbjms1`|E37S4X1pWSrG5#as72kG!qby~Hm2 z>aTs%&tb|^B;@7(c@F*h+de>uO9OmF1Sl*4rDW1Q=N?04 zT%9l{y4R1uvEx1*Dq(q2?5F3q9MRUsdCYGb6jw2ThnfKeDZdd~*V{Ggu?!T$ws6DE zRZG!XS$GZ*<)|bJ5+9r@Mf!G)U*u24s0heJr|aip)q0&)r*i$TZR>>J>K?v@2Kd|* z1MUOE?kg@nQc{F|j-LX5c2iP_0w;lo+aLYM3;A1QA#Vg>^jRjzJnZ|BbW#TB;r8JX zS`N9}ehg{7-S|aJK7kX*6CJhgcX6>*a?p`pVf%{v zvQt+YpFppa1)AT#Uyeh)Q;VCiJkp!sK@M)Isei+12nr#p=_aXt9(MoT%ryfP?9|8E zx1Qb{e*y1I&8TDmQm z`F=dK`oATT|30d31gP7XIk`k$c$jXtx93@;0&@fgH49uYI*`760Hni7__AhFwgP9) zWs~rv|LSCCUP{%BrStfk?T_fA8y$`fy`fJ)$Ok92H}<^~Iv+A*o&{6X%}27h9QSf;&hs*W zF)?oC8}gJ!iw?N}fg)p`M)@Fn0;n>DmbWLj_6=WnKf;2^-SF>gu#YzzWcnT6Yu8~_ zoIlxl#sx~wjfvJfE!lykmPUauJ?|G04BilS;--Z(-R7cr1RVoKlhu}BESKzAeEs#t zzBO))LPjO~sPib);YwCWYhs=KNo)|;M!(tC5m_|SzgO_z!KDky7V_3d5wh-*YZAGf zyKW@l4@QAZ->$O8XPy zsw=e{CSrD(?_*ZdwgB{Swd8Brwyq(yS%fdy6fXg++HTe@&!&>6M|sBB8$MEP7x-jk zbH2fehgUohYs8>~8Ass@!l{(FDPYjY1{P=u^=>m(WIiCNetxcBNYrmABPwd%F0g+0 z&(#xs^&HdgSW<`26a&?cj7^H_b$cP~@++#;t4fo{zM;IS*?(}4wyay!-#4gr-*`Ui zw&)TwvNbX2e8X{=FS%#rtE{b!RortDvy(V5EJkpp-)&dg@=ecT3x}6H^coJNdp?vSfFd}@QbFE#OKetlX$A*3I+q;;nxHOQ6KhSwcSCxRBR6x zPwapBK?0+;r#gy|KgNr~Cnf@BX*I9V*;-qwo5LOwS!)yCQc?YhE>GT=zp z!mF}GSexr?hrF5!eYFR_u;NQBZA36!p^-afsHQBXlv7#xLgmw=37%6R{-7wICZ1Q- zwlbfj+~?j|+aA>8#4k4w&IHAo1+%ty7E-iv@qS{kLUS;%&eC6$Qxz!A!Y+;Abfg@J z54p`wKqDl)5N%FkPEsAuROxJV9zd?GV7_7}F<{~`SL6Sm?6F=E4AZU*E@l;2ba94& zr8-@IuRu7Xx!&fAkD6ssFyUtB1u)ozEpn3dce}UdSey|5PF90+>}$h$T5JLe2h2Ru zCyO{dY;~TSU$kZ^3*~K9Mfj{r7fRj>T>ee5iM(LBs7QfcQKw4jX)sR?Oy*bhcfXhB zSe-Ci)}dN@qsh{A%)_rgdGGPbxg{rmds>?a!@V-Ev~r%zOVdio!|$f#%j!hu8IhYn z|Fi0rxoV!cdZc^p-ME?`$fi2a9&4)n_-G=bFhNqU;oY)1A#l-l8V#*_Ev|dK>_oNn z?8>MnlkQBRmKj=!jid_gV)8VW<GCFWWzqL#w3+OM^w8|>k4O?Z@XQqi;J;5_zM?4>Wx@f_PqRQ&bD=dZIP zBc<$d>bQVZ%77$a&dY{QlQmsQrbB`k@LuKvzk057h+8qw?c~PrCRxZZaUQ#5gdQ+h za=hSsByFCCgEo2Q()^Hu!kUj#;M~kO5K7WLKa#Ue;%yJI44!=lN*|?iZJj1d6I*p~IvuMKk01|x7=nS~ZK{ml z3~zdK7cc|b4UGi~OT?7bh5pty2qCIu@1rLzX$mLEX)Z93LKKcu{y^I;1IfB_>$77| zoL#2?9Ycmch5GaitH`c%g#PLVeE$djix;fS`ycr8E&ku8{(su9ZoJa%Q6>EPtQgn& z|BwF&{MEmnnJ5+T{nh(Sh&1j|PtQ77I{}8ns~Fhp-X{K{F2LAV5tj!>zV|bbPB(bQ z*%6-rUZW-BVP$^&ieIvA{6w!~ywaTYhJK%2%p7WpY1z9Un_pRr(_l@8 z)#$kC)T+d{$u@IKy2j)0r`a_;#@Dj#%Ek5w;POyGpF z;bw*UTa$2>K?hwVhGvT9RQMEJZ;5y4aFVcK9kJq+G$9@+ zDU?SOEj5+<==;JOZjfB8>ieBV28f)Q3w6O2-TF-jCp}L_d6j=G?_w_9yE3s{9+vG{ zKLcZB8S?>u7pqxRLyvWu29@_sId)U6&f$-sgH;9$NUDhM@UQSV`*$_*`Y`EnNoi-#BQxELp}83XTq6$z11L;thF$&{-5?M;}CDeMa%)X@|bky-ZZE_~jpX zCOu1y!(V=JS;<3Yuh_S@M!6??-By5E@Hcd#p#m=_qXTi*aR{P=!q3QrbR6y}tISKL z+&#MKC$y7S801T$>HG_PRHmsIu9*<72_G&Xvh=S+dyicf>!~f{tfAsG^`u89G`;Y$pNB{n#fB(_H|LEU;^zT3V_aFWH zkN*8f|Nf(Y|Ixqy=-+?z??3wYAN~7}{{2V){%`4@MldS~O~td&+MGU6;Jk{6%|)d_ z(#&RLG$fuuffTZ$q#CQa9Y>dr;ss5Tsht|uK!s5q3pddF_#vo>3*f z2$)`)ezfZV8a?B@8F7Gs#n|`B*7oAe+H`UW%Im*3A=H`2di?yE?l<7OJLTKBLiUMB zZ>33T*gp^`KIWeHMB%4-=3kdP|6EIj@z8RR304^*qjgtqnDHaDRBpv>I$(HirO ziKyf-rlIh9hC)U1XD_Q$+ra|lCoSowo8*o6U_nW%Nu;JGra~7F7S36jJ-vxgpXZ() zy?s8A%yGU~=FbWJc=EXpaeUQrtk?rJ*W~-onN~q&2scb`U)t*gPE4>G&DW|!d~E|0 zjxep`cVK(sp5a1>ue-a`kV;%B5 zj(c7yW;3+n7`^YyY>y)@*{xRMoTw+wAS;bp=O|jJTq|8+cj&EBGfJHNezY9ujY&mZ z5!K>bu0h$v=)?b1Y?B)6>iaR=(#E>_F!2tMWq2c2Y0vVSAp7A)_TsSZw>Qbu10Mp)t#I?;5Q#4{T`1vk7&}9v~Jq#ESyOQL5dd2$mO{3tfy>XU~xGzH7&G%bn2Aw#p)sUv`59N zVm^$`QspiXOhqu)R7p-tG9ec1KMjh`ylN&F+HM$8G85g~!@i7=lNQldDIFa}^8gAe z!99Q#%Kl3`Gv^@#D7|%CAkZ+}Z#a=%W<8TYO@1!OD(q7z##CzftbaWZ=>AOvEz}$X zWC`0*Xgi^_;>Yu|^8yR2{`Ob2um-oMUjM3AYsSvT}u4$Rc<8~J(KSBA4O4Sg7Y1$1=vha zodQXFfK=f`yYq#KEJr%XSarn%mEIziIs+9PE0S71BHyQvk#mAGv%TB`U*EthC7}S1 z_aXf6Lu4zXTz{9J*Q=|m$H$Mp?zfAfl}n;GVEL^?SI0o0Kkrw5&o@iTI1q@{M|3{l z6C(A&ddubI=mvu0>cRO)rtL{{;B)k=@wk<@-G4M4Of67pc76kmyyna7L}>N==iYFOZrF@p#muqHEhS^=l*klflh{{Td7S1W~>}kbRd~PtVZZ^Q~(hN|Dq`Nm8 zYF=205H>Q*^7(#i@>dfhiiBwK)>rn}h3qOr@rI1zU^Zp4K#K(&#^h!)LMz)w5ov5XYGXveHW7J*b_x(b-B%B33kV>Uo(V@bXgir{W?BvyW9)eX2rLABsKROc8l3LG1mkVCuZ%Z0T0= zsktL(=1kyXSOy-TMe+{?Gdzu#-!Jad#yB%Q`?@1+$O`5h9F%(wD0J?m@6t@ObKyK9 zVFD5k&W0Gbx%{NABS(qjek?C#8%H%ySpECe-fq645z*~#EN8auIdg#BlZi7X@*YcF zBVTdo;l0IqIGmp1FD4v>gi<4`@l0Kmu5}R`?Ix3HXaueIB`K;U%kaLuhJ#SCkQ&dB z55a5I*NShySBNumphw-)8!9{~ak{MR8T%mF@Y4LN$qKhu7hNbEYS8(9>Sv90*{Hni zkmPRZN1gg3u*N03DoEigwSn$mZdMt&!8I^~?PYCda&#p%c@ibW)rL)ocGfGmsfaWI zCXV;6m;AtaAN4Fu3Ml>bZs#YC8nk)*Osd;qUz#VpTH!E%usNZQpfKd(QrVZF((kxg zZcO0A;m)S{_NVWwjXi9&e!L@htchQ!M(t>ocHHlCJ&Th50Ec-PKeJ%+cWB4`P+p1F*7 zxDz68_PzsjM~_?B>9;OLqcaX;m^h)uhtLhAb4G7utwljZ6w%R6If;(r&5pae8mK_8 zpb@3x`?GO3jd_U#b@5u+48wtVM0ONYqv@!QxxKI3eNaiQ6) z6*u0|sfOqjL=vR-bFokZBuK04-DP?=DB~yob`6pG@y0EJ*0#0s1pg-@{q@z2-c*NTlQ2+uT*hl>}{R8?d8P z77;N+Q}SCSWuazAuz5CUlNc~KtcYi*`ab(e<@ zZ4IxvrI-H1rCi2o{RSpb1nSok@v666Z>Jwd69XULJ`M7uQbFMussSustInT)%B83Y z;0?=!#9Q~Qs&@YB2sbh}qEJqlLgk2;x%Lra@+ewnX83pt=2s0Yrou$8whP&ljzFG0 zFIXbHZlptncIwJjH-keMr7xY8erjbls&hA7Vf4s{iW=HS?P=LvyeOp*EM_N6-!Z3i zYj1R4a~+Y+;6L(rd(DH{Kycdfu3ssK8D_nt)uK1k`WOxp4YQbqq!-&Jn6Ac*J#%5n zc&jj)?k&I?Q$fJD5B_fkOA)a}J-rAP%7|0)BWcWuORN1sgb(Q3u~6cse~q5V5X&13 z95e9`T7DDT%;BDYiAK{8A^v`uTT;<3?X5C3qHsaP>ll$DkUPL^aczJxVVCj%msN4T zo_Z&jZHv#QB1D{i)zsQyR+wzp8GoI;7yXdlQN-$7-SHKcK1q$5TeWJ-d5bARPZB`S zD~xDeRBRWsEnR9qlQ0rbTm-1Nl7BPnT5+nAY^B*a>8Swab-91bPUUScx}v>BMS-hy3erW*%)mR9}a|Z*ylVKw_O{4nDNG1 z2{Qw!rTvK6AG!hHx*14Fve!;_Zv;m^8E88ft88^}9I1?h=DA-7aOts=2zIPpKTB)Q zk0)^AJ{3`$(@7Q^`vwheV?|_I{8R)J>c!gD_tpAI#wIb3Vk>*($@;pud_9Gq>m%=V zR*e?*8#Z<(Z<-X(+%WdK7u_gvHjOB`y2N#Wi};Xb*Uk!TyuM!+;*rC!_bbsMkkmVe zf7q<|4k-=vehNoEgD*=PzyghJ$k{xq=}l(~fsX1;_mI{76R^qxJpaOv5ZSQ*M60Ra zW2gyYiL`Z9u*M|3*d7k;_WoFcfT8P2G2nH|0z~{Ku^cZ^hFoBEOc}EBKY5u+t8$Ub zY_jkr^;|48;Tuy8#~S^g+br7gIprvAd@@N;Ow2~OHMYW@KNWyt> zWvo{(?=AkAY=efI;$#Mhetb=!wR}Y#5I*okcWsk;tBD2t{WnzW&qCWC6Foxp3I-}O zQ1wsilo7M*+YiepodNHBkz;uIYt8xfCKN=nP+a_Nuk1G1N0KaqF9c^Co7pB2C|jGa z+S7c-*JI7?BBj*Q9B(B3R<08H{LV6@I2bG@*;Zt^tA5~(c+Rni(qk@#F9)y9I%JN6 zad*XV$ht~wy*($MAu}e49t&DS)MhFhig8ATBSr%I8{ZY~Nnp4g3~bQ-C@C2aC+_mP zT#=HQi%=YoAeTI+HD|g#9?<#dze&LXdci&OgXf7tL-j11&21d3JBpacVZ%(nw>_9Kk~LpiuRh`9p8rjzDOr~=$g zTnB%k#q&{%Z{F2yZKnsJuvHi}t@ETp=6)bNubfPO*JTaHovfW{J$LVNJ~jzH zcwWQtv21>v_h#G%{Ykvs`?ez%6x{}ai6mK;uc5YTP7X<`x-}eiW>nU|JbGdz91OuH zuWp$-0lePK&&IHUx9TuEJ%kqup=|6mnfkrl=ic$;bOjC+J3k>y>lV(w}E_3SL++P#n! zP~Z4#6(OtB0KGbzgFa^OEGD}MscEXsX!sexas=pmV83V;k2hToVo5a^rt!#g`-JO2 zlrf}Q45uDDlT*PVDtMiPpad;59%fJpbAh=&cDeb&I3A_nNzK=9DFaZX=8}!|(M;^+ zl{i|HvCq<}gsa9l{}-1&FT1U+_>Q9hltiV0@w!uM&rUhc?`p}WU0o+nsVG!2gjcJ87Ua_Gc)7GZOaPm}w zs%e^Y6owMv>L`H<^f)dipnXa_ivgm>tLz#O6rR(uTMgcJ`rxq@tj#H~2n`*TX2=bT z((aaIXrsEs1cU1KRF*L=j*Ae#$bg%|*+N(?8kx^AWhk3 zK8(abh22sQuwz&(8WUOj;-icOfISU=ADJIzI-)LO*TNYBeFx%Fc|~;MfqKFf7>02n zlusLMwn6kvUbHmO0RLO`U)T+X2d#CKriclERK=@lT!yz7NS^f3IXvp6?F; zH1ugQ)YuaTJ&rO4qx7jVE<|f#oNf}113*5V#!tK-F zDJPU!)4OSFm6O_IBM1n4PySy4J)7_IJ5_RFv$`qdq%zwxd1guhz z>ipPh(;~pyp(#X_fvbBQ(`%wQ+_n-U*`c>vOwe4-Ifd#v> zGoHb!F!Uy`(#KkzUy1B8CahxB-O;%GPD!=p#pnA&Cqla{j4rgqMrwznh)Aud$E1g7 z(!+Adg)+oW`@93;<%89q^dG~F%9sV-YB`6D{t~dSc3^BY(^f$6H6u1=>>{h za@^&z^N8KLEaNiW7p0?qI=9VEWTe%wlrrVN8{!hAv4MB#)t4#aNLxp zBv)HFMjU57#;)P+gXc@qyf1uGVA*XE&n^?g~X!^K4`gUMNe_Rav=&GX@d5$G7hHh_V(x4Ck1xS9Q6_M5Uy{=I9b7Pkjc z&ep#9O;ID5)Tr52eeWB@oKsHXe7bf&4Wwq|Luyj}(vS^#LRCao&%h}qi?-*<( za(A*faH0-vVbk$t@a3j;0wclYc0agNx=$OjWSetBxS%qzc{AQ>zu=JGT*KUF+y=X9 zK2^@OB!0 zejW~)N3@7tPwrnbNBvTDA1y(wl zS3LbwghMFlO~-S)4IaWKB3{KC%a+fQosSIzE%)3BG9$hN*n)AO`uPY32RF!WQQgHs8N!7?Ap#$a1?X~=J! zp#0~@*ZXlbMPqEcc0XgYZW%#kE_9hLu;#(7;GDy2h(gjEY@9l__C#5?*V`OwAR%HU zR;ak~Gp9dO|L^Aq*h%2HoU7ly@d^rQH4Z13F0=eNdWI6{_{LC!J({K>f?WxULf=bP zcbtFQ)uK+#=B2%og4gQr(mN)VuOCCzp-zBbI>y$01aA09b+>8#^mm(+oKX=Kf=I5| z3;SqZm()lTeH%=TVZG}XnOtHrGw(7;_VmnIMjhF~lwsP+1EO5tQ2AOKvgbNTS^d|C$(wNRW5ZarbXk#9+3E zd|F{hoxwFYJ#?t;*ATADm-{i`vbEyaI_WZ?pkkd*z__}hL6WfhwSmR#A#Mu^OyaTonSNkcYm%}ZjNRHo2Ptx)7RITqq zIoj><=cFKypStZGPehX}$kw8hrjRaMoP8R@L zexm6Xxj{at_v0&zhQ>UBS8h>(u7*uJZlQqfAvZxVF;AhqjlS-x%mW8OKjbLSPsRXh zKIt18Hm!O)TV()4K)k;sYw;)^VBo@gA$J#&v~~I~UP|6>yr?*+!f}YSMc-kl&hlYQhe3^ZKr4ugj{ZfL zcfc_mXIQuaNz8)(7-Y>u#iVwlGDOAvg;cT^Aj4JdBeUY98YRX^NBqaiQirg${+yq+ zXT;yZ)u+c9r>>_>!W`s~m#bQp5P zduCwm&`9=o(POA};ee>t!5T5UdhQ-QAL#;(Ugvn!B2F4nW>VH<#?ppOj#Hj1cbio- z2XYNDd^oec)~TvgzO(ibmAjW@q^xjb1>Ti~U(;G_n}osrEvh(AjH4Q@LmMFXW)8#jeNEDjs|KE395f2`N^B$Xr8NWLaAo`*>?txPu!{K0Es56vy0%?h&NcZGn z_9p;BCv4`u*1}afcBa-FX?bJ3*%A)6<9wKqEN#7xB~Ck?%CD$~`1or^0q+@`W+))o z%|uNt{44OP$KP{{Go6Y*@TI(8WsWDgQ-fmjTygF->s&DA+Re_4PghRHgX<<*nY~e@59xJ2_%I67s%y^bJ<8PT1Mr#T({xY+OB&8zG$)vg1k1L2%9hq4C*R0 z2|umk8em`dxgThv(E~dUk|-LEXe4GA2|*D#7##7-au|lF`>tr30}w?0utL#P`{0LqA%&tT z?gTwF5_V_{$f0rYJ7b`RVqu0xK@4pHxf0X2QLsYWYCfZ8XDpPE1__F2MkI{TCO}Y3 z7lRN&U2vc{Q?d^}s23P0f^0u@&`#pizNP8ygQ>=W0F4F!YQg)&StEa8@Sn)dwP(2ZD7i|NIhUp2A>}Cl?L-Yg(6ajq3!1Dxq&j;d`2BCRE+aG{?#=-Ig zr@i9Yrwhq*I5?iIgKkEhL}3V?E#P;;YSt*|9RsWrVq?3&?u;6=k>_0bHNr=Crz zganvw1EhicJ~T7isJ*S({0`%koCCLH3FG%nj)&GmzM-X55I*5xZ1+g}l%k`o(6LlC z$F#-ek(dLge$??F82=kbQa4jcMH#Dfbf61ku%vmKDRC>LVs3}~a5OmM2<0K;f;IYL z?Iiz6r{F_)C%U71Qy&p<`4){Fkg~p3*WUU)eopoDA!{q92S^H_Gs+H1;NK z_nUlDlx)sFAbyH`P9{bXAQL>@bPP+l{|ytFlHZXRwXB2_M1&U3TC(k2inf(7ZdRsx zf5bjeMQ@sj)%j{YN#M2Cd8AONj+=2?KmC;iOe6vd$b;D(;lL&wKZF6i|r{EEjXV&MbN*tb!MI3%-Mee3@UB zVo@>~yLFAZ%kQn7NybDCkMfokGgkuq$Y)>dSoP}08K}ZdY_XE zmo4hjOj>SMnOP2!+}{3)mnO z6QLJ**qh>T5He}24KN1awE@qpER?u0%YxC?!Z`{oXK6`wXPxd`efuJ0|ANjM@HDU? zYXScr1Xd>!R7#O>8jgm8xiXUvCU5Q7&x$2=vWTq-3+zr|>z_AL-Pp6Hera-7$JL^w zd7V+8c)79551^Mu1)+`$Kpho)`pE{K?i6$yP&y^4N}AlsmRUWh%NZ)#&i(BRZ9i0- zm_yJdZsbRm^HJs8r^?ynwG^UM>kl&PSHaMN86z1j7bZW(F;?0mM&IGD^h}2nmSLuaE=ruI&F!iV8FdbUc|LZ-a;XRLa^6SNm90k8rkRrJ z^&8+khYpj3W=W0FgZSRDW9wuwA%rEDm@mcBvw>bH`6ED8Q_zqn8kZp#O{C`wq zbfl=MUKy){M~j+9m<-QWw5Zt#lL4Y5Ma_MrJ?=-;G^6_{B>oD8M2GLT@^dv`i#-?g z#;Ws_mDw6%Z%|^|%K{TQuAMQS%jO~DQ%>pXOEbZtRAm~C4pz~mS$Cf5~JqR_E_T%&xNt z>V*-{jEY!RJwqF@YSPw&(`bmu6ib+Wq z;|EPYXGUsSuV_9g8BL*{g8nsX+<2Y>$bP=)>?AcW&RE6_*QloW7m=Y__dgx>1ibG5knB0p)-k#`9Og+Dk>#9?I9GOq$Q=^pv;k>eIZgE;VjTkHvytQ9N9#a zfr%oRe@xKxROes~K@$PkN4nIJxQ_o0O@{CV4)^PL5eA1%4X}R%jBXdP4ub1>GbVY& z)W6vbXznN(aY0WA&nxCm6oyN>G2uA4xe+F^%{lL6 zdk%C{lwd-+JCSB*_RmuUmvv_H(x!!YB8VbLPHfFw^RJe%jK^>=OCo}}EeMGl1DL4l-i@Hn&L(5ljvTqH*iHAA5#lOo>?B9H#qIc+H$qG|@R9 z?Q3EVThkC@3tm$7A_$@S2j)yr)o?uNXECXU?OdxVxq(0dk!jp!h^INT!o#-3fpf-E zsZ$Z0+753;#h#P5>_P7n78yH7&}`NuaLsGj_-cd6dgYeLT(aaY!Xio_Uj)s*wk?V? z%_+~^0#Vs|zvYZ8S}xeO0tM-Ep8z$@m0~6=TAtAINzmg!9HhuZr zlu}k?V%;xEymSDUY)ZAFd#kJL-cE_B!K>s9a9w~mO1-~V@PX_>H^rouj2s_Bi;j;0 zzn{?*^R9I>>B5}8K7@5D33!?H2Eu2*dH1@9b$Dj45}9$q1%~)?-c6q^M0zY;3Y$YZ zXbWwTzCmIrxZhZMAjee|O_nsiT*jFJDa3IAtoK4(ew zXVg-f)c>K?ZIkZ*!+iR4-|`8GefZX){oF1b3%9o91E#x&RMdb#o2TTJ7ug@JB3M+C zW?74f)Iz&6v5{!*v}EM1xK>kBXY7nzL9STy4V-Il#j6i2sjES0ImlK#xI%3Z=LSaz z``5w|YL=)$2pZ;LLXYRbxZSaJlN+nii?CkBAW6G;3;+y5xBP!PEl#XvJjcdq1D0ss zI!M%jJ>)4g4w|*?Acr80S|j~+B!p2YWl)31^??;R$V0DwtZ6oF@n!sKkgk5Xh1{VY zd*E4`&=SKYbRlvnbSRGEK7$k$FE*^FSqZxKqhi54sWV!d04q*GhUx2UV;# z^Xxv=J^u?>03Cu+a8Js7nAilcf?4Bpp>6H8*Rti{lIa|<*H*s6W9atgOfjWDi zkP~=pxSgvNOHrCXHF?CCGd#3FjEjZm)#rckE+U_|9hw}qa=tYcF`5xWfv2++^ik^N znLVgkxDy>jDyOh@h-F%c%73W!M@h*Ru*Y!sY`McC#Lqt4Y$XbjiN*T5C>c$a_E(j_ z0k#&HkRFjh5JLAf#jORXfhIcDz?R|xp|s(hUTLQgsU&nIv}Ap{$UP8NdL4kq zY<1|h`-F{ns|1-tp$gr4Xz7$(iPW<=tuTnpr~}=Iv6FSKBWzQKNkM*RYhJHrtOVD( z8Y;98@Uji@(#&%zR+JBV4P$JDWAHZ*gS@#9%*}|vh{ND)jz-ut=$S#f&ALAN1&WFIFJPZou zc>Ku_7Sd?s$+&36ePB*Ttgwh%#v@EN)AJQu$LZ)tZ(7e7;ib|dWOHJJg#!<3uc8ie zC*Z@Q3XEdC;!d_`y3Pq<+P18Sd#!m!!p?-as|z!y5Htw(Z3Vm{2Q zUFvi&Wg~F3M;tWagRsS^)Bwk@st?z9>>-Lr9lybwI|WiwwkR2l4Q?;kZoV|Y86$Ph z?wOkQluPZX3x5Gj=qH(&Vi3+8wS4wqns{7jV zy?|;0iu$mbY2G#^n5}>w?nZ$(=iDL_Gn$#2F~eum&Bnd9<@(RYg3xM!Z;!&k&KD5R zfffa|4S!;#FxhYo(awz?x{_oJ9PKnGZ+QL^?{5K^oWa}4XOo{q&PbBgvSQ`%G?`_D z$f;`KJVi+XM%V)70$*)0f2sxlpznlAcGfd9>fKMLHtDn!?i)=)x&^0L$u%-l6sO0uqQ+V^2PtJRPpsDDpO$doG~%zwYVD z)6;3I>#B>-nl28~HC=D@8d^>BHN0||9~f3}k*^=-==+_K>JwlrT+vV^4;xf1s= zH&IsjvSAL!t0}nzOkvs}fx^*aJyA_H>8G%^pJ=zlu!N=Spbf?KIJW=Ko^5UWU)dcT zDVEy%9Zxgu;Zla74E8TIjN`jyi}&_%BV+-H!!0qsomD z@jK&i;Vvn}+-jGe$Q}MNB^MgV7(8)aw5`BA56+q43Bx@90l~FQG{Fipo)sC*h^|Dc z!*rBBYnPuVmBen@oM&eyfy_8cW^75J>^ziMbN7WFa?dhVXN`-M0eX(!rm{UvPxJY#IRY)G7lx(IiBIE6;LFvYG5E|BxetBDtZA12L`Q)bhd}- znUWi5Et#oCIL!2jbIX8(L@J)`_%AisgwdWBNB=T0Qrva z&{pgm1#b9|+)(W7ob)JlE#=L(J-1cr_H!oZB4rozInQ~uz7iQv;v$*=sIC2+b4D-4 z19@OL?6C9(`+o&KXSWgO>JPiN;GD8Yot9e7krNWhB~0_z1!j8Fl07FA@;8x-@;OSq zIY3w2qinDlUdxmVnBF;jK9)U9nJ%XE9gFMU6JCW0@+9<=Up17{3J18O@s zN&O1L<`JmX3Yd2G(hCKd;o*Cyvx0JS^E}`v)pmzDDE!mO26$45PKH-KvZ9`6mRqzBGJ&Y zVUxqcFHKy|S!guDebF-Gdii}*n zY3?^hqB{oXFU%GiVyDmt>976?0AZ$rtd_yJv;hh{WqHLDnz>SH0~;1NKuuM9C7jq$ zAo5*9fiRjZF~k~Q@aN%;qo(y}Rws9?GG3%|tzJ~r6`9eD=84)U-6$JRRUddcQiN6^ z{2QUhQU~hUSv|7DRyHBZyMvUEE9Yl)@ndm(T5^qO`etYeCc`b z9Q-~cw=QFy5;k3cOTDwSj@tDcmO@rJU9qEKg%L(otvuX`#_Fg`P&=cOr)Q7^(Y(;7 zF&x<^+bvS!Dk=y>O87i4lKE#j;FiJsPzf)Xz#Z`Zov4ZOh#7eDn}> ziD?I>$t?wRIXKUJkmd`14}jh=20k=~E?HKPlBKn$^aV5$PWubj8US(m7Etl1HIYYH zR%3XRw+?MS2mXc-YIDNwS-B>KkeHVVES^fq7x~KP+n<}bG|(WScShMd(%RM$SaC#) zJ@3;V;281!cB_!$Pd+<;dj51n!5<{*D*jjSUt^Uw|6a~}>?EGZ2HmIDPKh+!dwmMe zz>faeX=B$XpPfHDe|n<&{OnZq-m%O-n>;-w9&E-IMcp$dTekVjTtnbJh%(i&)_M{L z$tF+<(0F!N=DdaN=j_-A0qh1`3cMn}TZxkWZg|mKI)k?JA=7t&1^eQbCdM4t`&;Eb zjwYL|z2~kiW5#v5U?4)=q!K!_=ETVN0S$S^%G5rq!;$IVI=7-{0{GpWFY40RyPa^V z*8M$WqEB&p9Cf0jod5R_}T;wehS|X*r8J~SJ zjYZvEupBZ)Qd_c5DHCFDs=8hmEYVZGbws|?RL7xC$x3~rYjIG07x2visgX8)+9vWI zUUx;SWt7ySZq$u2_Hms+s}5DGm1>Nn09YyDGC*Mcl#n;ItU##HB+2pfK@`29NQRv% zML`#^TBG{beW(fbM}cgmcS8vvbWLa}>Rbmvc*J;} z)AXj((6G= zJd0DN-{?#Z_0Yt~Tvb`KGyLkI$bM2+qI)3a)NgrOHv9^oo+pSx|^>)6mf|^0Hv##ei75> zVHu|x3s0ZF&`v+eS3IL-w)V+)*Vq$~l30GvO93)Fxu+${o3>h}B>#Hy?Zvwv-d?ca-(J`b*i*cjo1L$ zb^W#g)kUZUimnKy>pPGfxv>LmBiGOe`6{Eb;>j18NbXeMBMmcsPquoRlZ)@J$xANp z!j&_zVrCnqf|@d8N(ln-z~E}y=o0p!5YlgQHDj}FT~=F+^+7W1?o4$T>Rk>gVlnAy zb9obhm~0O>-ft}1^QLaHDajlOc?BcA7BzXG=t}^eG9@>>curpBvc~w3s`DB@P2DnC z7rMrUb{N2{X*jfeVY0T#D>Rt|l1h%)hw2YUXXNO^T&jQaYA%na~RxU@HUN!nuiz=54b z2qC)qr}LEEpUad!J%c9TO7W+0wfjjWIeL0DCD%q0I&=GN@AFcei_$g(%=}M|$cZSS zMG4JG#;68-(~M?~Nd%`}u(OyT3=$QHK_avXvavYK69yWgthLX1F3N-%m}9ubJ35k8 z6#^i(%orb)!ILP>gjgx+@@4!unI{n4lmA#Va{1CQcn4rPC9UP7%^2drEbt?k^ivB#LN`K+D9+Gd3QOg6W_eM zG<{GX#eAdr&_q9@+)wjI3aX8q`ejE(ZI1Fc78ITBc)3()mI;;A49I(bz{eTbPpLKd`&zXE3!%94crXy@GYi|+iv2770Y>CCY`$V z%j>qq^aU%HRsk``fv_ZdR5u-qB&T=(2FjaO)6Q`M&fmSdc=_hlbd~;gDIO-3nADO@ zc;!18BcOI7!CztkY6_1dkMx+Es-+r2j4!9zbMlQub3WL?j0;nU%CEhY6g&jBc( zG92rwWbDkfG@XBZDWWl)w7MFQDT3-6P~7)1Kt9f*9CO}~~ zsCP`7aBo|H)Zb|*jP)`5l0l60Aadh{Zu_8n_f~iKS8MrWHo><~inJO3PsefGHp=-8 zMC@QjIpc{NWts~$7Kz2Ue0QEV@)4xJPg3uXIyZM1&NvxwdpV@x%Y}e!PK|9F6vcEo zKW%q_TnAdx6{}d;An7C`&vk{7uM3u6S2VfvdPN@PgI~RTu!F2(6`qQ6? zCQB#b&|r0FP4=~9apP2 z<4J`Xg2*~W0A7LElpS5~N{fNe;WLu)J4WX6ucfGq28kHMVZAgpEFCtYS&NcYA2ALo zXrFBDCAhmI5$ZhY24lCNy!64*5*^C{p&yt*(mq%dmXa5i&6ptNq7+#5)9*;mO~pK| ztVk_%o0NCQRbT)Kln@dEwAPR4Vo{qFRWdHs>I7u8EnD2gWvi7haV^#P24zHG@*B{Q zUqeHw0S$KIz9DTvgL~kY6G# zt{9e8xJU2Oimu(ftzcV^r_*C(A)+L*qNU#R>id1x26;mZiAelPh=p1g0w5lmtP!e5 z>XWF+p2G&6II-(U}xkT_N$9#qg%h(;KPJ6pjF9ZEpiE z9znK2e?X=jV)?pz`>wtmu3u90XQFl6w3pf?A(O|aYwp;)sUvEpt(q7+X66Bt^gte# zO#8(h$w{3aycfKnxj@u2LJ;CS`9#PVOW#52k0g@ESrQ~CCogQmoe3bi%qv#TX~Irj z3j-^bY{g0s+^GqMG&yfenx`6MFkvMy6852DrJ9ST*uNwvU-**LHR#%)=Re9k0dI;A zREFCPYj%s!H_gUgVi!Xysv0`}2zR3&s8IsT}lU?Ks!}k3$ zx*`cGtf~G6u*brT2!w8+RAFn-jdr3)2b^5wh5!^&H}7eX0sO5knce`tW%uZ8`PDmH zwvVE2>iJR!i2b91agg1-GPlw^Z+N$HjA)O z+l{-)7#pDgZ4Qb(W&<9aH?IRQ_Zc1-r_d~aLR9`8|$}t*>E}(Z@Xz`M60TK{!}+4muvlLIliuc!973wNS7Q!G6!n_12qBh zY92j{k%|aj+o>aXZR17^$RDnZ5w*P=_u-kf#J1gsEzu%o7lwt8u7D9kHA)0&iy0AQ zW88<1b1=qv6yZK}B#S8LY>fNRkuRdyM3e;5k}@L5{;J)F&Y^jwJ&BvH#E3y~?b|TU z2o9me{b_^DtG1f#rjgD?teT5lpWAOmyQrQ*do9 zoDQ}sa^_G`!uF*>Hk-ZCk$Io`p>fPr(5hl({+#^R$@|~`JURW#$;tOmCjb8T-=Dmn z!iV3T{^j&%^T+Q`Pft$1|LVu?rtY8RBTm~Ky&ZgTQ$kB+n-e8p(U7Ko;32!s$RQ{!z2VVhd!Uv!~H z?A4~Ylqks?5EBkQeIyzLfd9l^D;dY@j0&BS@n>W7iP%Z_GtD%SDwO~|Q^U#3p>9?d z0hdSGQVcAfS7xsoN1$3l>WwJwCOZ^^UO(d@!i}0FI&y)3^2akW6rGN3t;mK}M-jFJ%qNhVB8%zg~)B4(@#`Zc2-Y zzOfl|p2hD*17K|@q0V(6+DF+wpQNm_rGyhc+)Tt?wR>>LQh-Sp;A|gW-P`07^ga5S?BVM%uY>+;4opR znOUx!S`A=DDw!H!z_BUCipZQ6aw#e>UC@Fiyz-PVENOD59tEErH1CWgr#igZowjz0 zPqi#Xy%?6?sC_!gs5GP7OIK^jlBDaLhEsPju1<2Hy+6Bx{CM)Z|5_SFI>E>W{0ZdAcq2<_dKm>h6e#Qevssg>q{id7t3n$z`(PM zXL`eHOe1+WKy`SRNjPXKZRbO4JQT&GYrxIZ@sO<$lYnBaS7hG`fNoyciit{?1f#f` zTInm=y4go4FWCTv+pqCR#LB};4;a$hW90YyX8qi-RguvuK9=L?JI8A7cwg0?rg=J{ znOb&N-@XV%3fP_63dOo;&5)gF7Q`{=h=92xxL{I1b&Cq=p*%{eLoC?E2Lb`iL4WKC>AA~7;z}H`1=^D1`R5Nj$&7DT2hScD$-!* zvFLNIw`4b#8~QHtq!EQ+VgrCSZ$Sde);ekktp(bzle2_LNy|0Xc;%IeiN=Upe_PcI ztZ0sP!b*l!lIsL!ijr9=?pSV96Vz#N;}Zgji3-4(F%hf~x*+2FxLM))aKtfDEc=%~o-H`G?n z(h)B(U``}VT+XJ|1Vq6ht;88Hm}eLxz*4HFOuh{{k;4Kd2>+mKnZl%zFC z;|((n`2?davr4)fHKI37;(yXa*}_=#$(mv3r6S1kikHmb8ekD|80b33h20+Vuwh#9 z>;fzCR`e?dIEhl|b4i`Pq0WzS(P2HfHzJMm=XFZA zH4it*7ns~5!MuexM7bSSK&9$dvn5M89q^B(= zU)5ZSM_os|g*wPT_F2v9b=-!H42mdEWiG}Q3wlm|*Aj5FFl;q?OvpJgD*jYYh{29( z-yY+zCY=|8WY({x>cpWqXQGk(l_Xw*N!_GOss&w&EMp}WXh-coZW**tOrYWly1ve= z`Nyg%d2g-s&CAGVR3e(BiAw;pmx)R=?~HRCHtoVv$Ux&FU*llDbQTR7e4?l>RJWAM zlhawI4s>Vn+4i>2y|^(I4yAW&t}Qk0S8e)HRGYK6MIR5}7V)VKWQ`)MC3^=iQm3;u z@vyF)DlUYf+Gv%FgwxxTh=`7{F&z*|3-MracXzna#nmM|Z1lnf{9#V}XeY6`W=yez z1~CZ++I`395Iwxwd-yA;+o2tAwitWK)SIG7LQkhqCXnW99i&$+d6`ZMT2^bc(VW@V zM-z6_CLd(r)DX8BOPjLFe7SF4w9=?!CX`9yme zAF~1I9o&O%?A{nmp?4b_8Qig305xgz%3b6p#cE|c5Gu9@q*WqaxJhtU)&={6dfOS{ z!xmciq9hmjL?`Q8=Z%)TO$0I86lfkN*=eB-TI-U(xVqF#`Xm=QvOT;vPH;m@TCZ}0 zWUTaI^2uXQSO=ELh(|n6)p-ENA{a7cog^%BD`~T#TM4;20`%~#Xq7CPBvM2zcS3kB zE1G3Zt!}*l1W26Pm*fhfaV*@)zSkDeBBR2nOoy2x-^w%K~dIdf?tZbh{hRB?jC5FN&q;Z6~~HgXDGF~0#<%}9JfWA zY36xX&U8C{()<>;O}jGcsg->tvo*k)L5L$-S7Jpgo`40A&mHgD%2uf`u@rGz6hsja zmh@!lJ=rgjfsqt5`IU%10T9W3C*inCLScI9)z`eq4XCwU31Nikll;)W1HC9WZ@+de0X*X7_VxOp`jf<|HjeQN#*LYOSAkdk(h3}(8o`s#3EgOFYb$I$5biQSC$pK-k(}C@c1;1yiE0Iy%^Nvg01S0X zzR@M%?fGK!vIpz~aY7K<5YRq=o*Qit9Yyg)DWrUB z$PSr7Oc!De45pipn)st8-q*cHO}y{=XVAnQ9=3H?t#sBlE;^mwdIOWUqSAp1AX!r~ z&IU?0v@{C5^MS6XIjb}tmGKF@Gw=OtNdq9{?xoc-XX6>JU=XvmXmusz zBLvj%ka0YUiP@2zl51va=D3}12Z2S?RvbHJ73G<1&eEIy^E;a8h_i6ausY}AKfQI! zCc=%kOf~tXPVloEkZ*5@c3AbfT`%Y49ZSSLD{%q5`cTkZa*=;Yc^2ytGE1d#3PGv5 zc6`D}FQjGYdb(G705tZOqNxKzi;~@QQOAvgPGhJmlrUtab@e4**vhUZR2f2V$~0aZ z#*kuK)v}GXl0ZPi9$nma?hhexVlqbTaWx)7uJv?$W-Qq2?%t@KoJYn>fXrV5iuV_veY+}hyn&cpF^=0=O z>8v0&75{O6{PXUQcLwI6vqSJ?j-OncSGc)DS|GXRePd*DvEo(5OqjFX84#b6M&j$a zm*xrJK*6|X_QBTLbQA#4v>&)93!{4}Q9CDmRbVLu*xM7GgJ&EEqemctNKL{%B#foH z@xMNHGwODP0Eok+Xsu`Spho7SGnSDbYgVp_k_$~%NNfyGB67)7ox7sFbupU2STP0M zbgw=r*%;8Sw$iV=MB3nlFci^C&JLq1v0|`jONNbgtEuba?aO#QYD5?#*{~GqdGG5) zG)02LXK4VZuwkRtT2JoSI^H-)bl|lv6>P|2#zCvR*$9$hNZqGCw14MWz0E1sNnG*QL5%3MSu*3ZDNclZ4&hVT!bWkAq(g zv%COlV`Mv;w%AxoJNaCa%bbW(kM0&*KsOSbXWR>oH+&?;p7!Yp7iDV(_tsSF0}rJ7 zY+gtQ?57YxV$(vPm>;d!A|lYLeXUrz0Ccz{s{B=RjXb09+Hzfw3%@tl81o`t;R0Cg zyUn7BmypLQspESNg3xP7bow}&?G|&oDY;>TQR-OHLUEn{tAq>KK>u%q9?MHPB^N~U ze33DihB50ouZt$DTBP)(A8US3Gluz$G`BV1#C${kjOP|u!><<;H+jqTLZlhYwe_HkV{rgxjf?ZqJf&U^4Cncw944 z=X%{6l&{@KRQ29q3fSi9-OwZk#x-)T#z`Xv7|p=(U(Q)s;@r*F{hTZ6HK|}Y)9|J< zoV)%Q&N10>IzpF{D~8)vR~!S^v< zNUX4HHi^So7}fho$vZ~Lp1|yQZiskS@Rs=?h()^~YtlA9e#vJgBQKUT&spXGF{5j2 zu+6lpXtKn!fD*;GGFBZ+vJ|o+E1E2MZXq&r3#T2(pD4tjs1jqg9$A|9Sl6c2%d;u& zirc;^S^Fjg^8)46y3aRN_cR-Tp+R*P4kx(Rzi5Yx<&w*l^;z=VIhUD!z)P36q?@TO zmbnxUBo*X=K_HTnQz5fJ##GF4dJjo?iC7h-SaB02Z`Q=l0K8f4Es4~g;jGTp^fr}d zzI;I{S}s_n=#ac#&6pR|biXymMwjk9wzWH3z1Y6+CL!+(MaVz`8FDVl#DsX7QlVHl zPR;zc`fRN8N_r3NxT2mo{NH`|Hon-}ar4a3Y(9uGOCNX|26>a@L=igm^(4ZyJE{__ z-+1}(z|#?f0k^jY!Kl7!5QrKD!c&%4YIB!T-2pR^U529KU$yaq&(VU4|W^3A5|5EodJ5z!D9%Qe^dl4?<&o)Oz?$Y42JB$hUtgol5w-9ueD>sQ-0} z0XMzn=}F28mIv^>L{7eJ)+r9i&SS6Zc>VIFnvYFIvDS&v799%+w4>+3iT(LN{b&L-leYIKIFQ{wduQhUxw&C|bxtYV`?o}lq!8+YNM%fHV7us5!3 zkiNHL(a>LNAEEsyv>eJLf*_jt;H+7{6mpn?=GJS4zs|-7clQwl_U?^#xOTqA3mI zU2khqlLrkjuJx1~UOXqSa#@$GDGJniO=t~AYAxgyV$I9vMnCZq9%;htnkl`s_q`gt zFN?C5c5R{|E#olu(t?&1s9W)i(>`KYq4Pz)>*~Cmn}nXWADl~jlV8cU7;@?qa#!G& zwk``fPucysOzG1zXhKZ3Zo8jUlB1_bQ*zB$JfmfXfM6CC&D`fViHp)U1cKV09FY@I zLW>fblZ+W)G0kDts1*cswvWa3$llH0MLJ4!ImY{)Bh@u8UA=6aRROi+uPC^D0z&Zy6Cgx23FdHL$<-K!TDH?LkkCx6v%;I8mIB{!lm z7kd;QoARd%irBWDArmGvZ|W#Q+mMLW3aim9E#z4oz@_CJyY(T= zS%Xs_=B`);Xg;`QJN(t$-qBF)*E%KLAG!c)4e+Z`m7{fuEotze^pSRM8+HEf)y2y< zucoVX7Z%0xv=BV^Lo{^iXq!jO+A&`^mAo|ml3!-B<5-D8WMa`@gL%Y2{1$_eCv>7l zcz3m?T8`0Zs-+o_re{`~ua?o-1iyI4_be24M^bI#sv|R<1ba{1-cubEz?FvuxcI0@T(Ky1>dXrS_B*CBNqxTlfrtOP((r z6fGBUk%R1AYs@aZ@<2Q!oA=x-dWL64e=}qLMD?5n3PWbp=9PXJ0;&S z{eDPqs7jtxvTXeWO|FEzgdXIOKueadP#@cSB zLT5!zV@f3%K2M2UuQHzBHARf(LW3kSqJA*zw`A0xU(h`rc3J^DbN_0Tl^G1mYM=eh zun8UP;VSb-N6@=~#QqESJVLeKf#-6WNFK*<;no(n&B^8Ui|fm_Uu4Vp>4*SouYx?} z7a2NCp*Q=HRg)zxF6wIel1l(#j2BQgo^*0Buu`!0Mi8zB+PhF28ZZ8KaYbmYMz7Lk z8h4Zonm%4*j`Qdq>gRUopP_Jw-t9MhzU9XXSl(&1g&)gLQ@3oiAItB*`Us);JDxPK zqyM}3u6WKXPAgH4@*A|CvL=*TR$_%a49u83h+FX%jCq&yO@d8cDCj^i=9jtBNXiV~6MED06Ojhm7mW9{TT zCMQ)=c>Q~O&wY|}w`l46?fkgyzk=V$Yc3%)CsdCf zZst;#g$2p&N-3SZ_<^Qr`5e#Sxt65LV8Xa5#XxiG#XhOlg|2mU^WqAqNb+x=KLJVZ zvuB?_@lO87%{v)wN&2YX#ht~O@v+6N7-GPtP*T=4hSw@ysy8_-z5qi=n^PlRPJ4`v z{gubYn)EhgOwkL^eF)OMH<=t}df;dO)*8j%93JMSx*z_w7e$}{IMmdMI_w62a|d}_ zLxFGOwKtp_s-n}Hz%PtRLNDK5|M2?ai&w8<+%USuW09uUnqF4UgQg1*Pk$+f{YS=Yu0~AbY2SKiyJN)u2FexN>?(+nlGvWwpVI!G18oF|-X$`OM{oxIDVWt3n zWhdqd6vjn|+DE#{#{DBx8 zqw|G%hFvpTbB3j;M7~94P2YoUr~MV&@gfY^)~pDOU$(9!s57k3M3Eb(Qro4p!yf%- zN~_y0VO>OquaKswrw zbRuZ?XA6E>TW|YQGJwipk4X7ry>w_UVp_8`Cn?2004g+&iI}_5c#lP@vP{j2 z$lWqn8dzaC+#X9bt!RN|;7KYT@&{U`o?_8aX%4s$L$H|bM2F@X2(AO4uKHufD~ggVBZ_6{w>WmOoQR~NnJI0_st3mW(&UpTPtf-G#yH7IH~$qFJX;Duhxhi&MA56 zgx|TgK3Xk<6_x-lc|LcbZMig@!2pi%#nKcYS8FWe0VH`ar=VTfVBTPuCA`O;B+!L3 zlJPr6<`PPe`*TSQhH}ZI68SZHHNL1>U6Of0i1zD@jHqeb_7LgKY>FA_>RKj{9C<-t zGekJ~TJOBFRB;+KfK0=9XeROn%v!)dBHUmjmf9@dbrJB`#Mg{UFgp9DtY!5@ou^p@ zaBQo$M=v%gU~I2W%`oMxqwt5-9NkU<8MkWa-8Lm{H(wS(Rh!P$exf;{btNVMLvMnu zr{sN309W}8aF1hv%(Voy*&b*1T`c7otlRf_4?Wv@wf(fCXlANv#WppkD|V*1tpPZ0 zqXkO7$XSW@n7E=*v{SPUhVUztq173qcD*EYv4H9#_c61SD@Kv=pBfCqB_mg_-argE zh{Q@1t5}Z7jd7UZY5_K4@RX2OQdEHGrzqof=MQ`OeB6d0lX5i(ltf$l)yt| z$!Kb-Pb;ygC|Eir?^;xPevECgLQU2qP{$ub8{ttw(Mif#F*Ij z#I|kQwvCBx+qP{dJGQlBo3GBf@7}84AHBLiLDy=NF#i1_!UJ17dd|{vJhbTn@UE^` z*mv=AQn=YIiO4M*UYxSg%8G@(%1pC$uR%V_=jj{GX+uML8O!A-Bn{Wls7^}Fkak}2 zwKyFCu~FnLY)R~27)$7d_;AQ(FV#BM8;UMt7#v%z-zVw6vgb)7%6v@a;b)Ou#M7uZ z3P;z8s2#;t4sb>9VlBmvt429fiIH%%KoYSy#xMkMZT)7a{D;^YDxDXX<9s-`Y5S1Dp2wAz^%Qreqw`cKI-rZWb+bVWA zTm67)=saze`yDfwHi~1;t5Uhzv-bm)p7_Ac$v+0K-e#DZ&qa#^biS3wq!l`XMR3!QG6&u&A^&fRctbS$obt7R%P zCqd(K6;j^2q;{Ry75Z*wg#d8 z)I;04y+F)z1m*$1r`*n9bHbB4zc%``L9lD}VC=0MZEsK3Tya977RgrxNH6(w-@~R( zncd!@b#aMjf&a4M3sI7eyHu+be%iPR23LIgnI$NX*jSc1@vc(s82jecNpQFx10^Mx zoYVoeu?k7$jg2q4UF^N$sa)J_3f%tj8TnnZ1G0U+Su`u)65m%aB9WHpJ6m%oa4NHu zG7&qnOA`7UX^zIQ+uoZ^E>1yqqoNwgYcAH8lO&p4nk-0GU_3%cS!$0Z@mP9U^Md9g zmzzW^h9hbjHhsD*x_+Sm|JR?#MuOOw2jATE|P z6IERUXHX-7r&P%Z$c%}|RCn5zMAb`L-HL&xlZXXeFQgFhRl0x~58>-O7q4ib86u@Q zT8v_A$jTo-;olK|kjZ`rlBt1k*tA5ukME0;oLh9PvONP6T7g7eP1j{Rvc|NW+EnLN z;~V`wGkl?#DQfdXI4w+*ge*w>A+Qde#Qhvasr&(xP?9l)Sm+9@k%jQRzaTVh4JGkm zMxDTF#gJWtn&^T3GG2lX;`1)X=hb`U^H?_`mN5@etjM~7wHRbX3*XVU&`^&Kit`6c;sb*}GB$qO<7DXXLJY`2Qe79os_?0FGRE?dD zNafmj*&c#p(V)l))$kK1EO^_tzNDiBgYWZq(8C&0ADGtO#azLwsj0&n7w@TN{e$F#1#6=o+= zQE9TI%oQ3Pi6YZu(EjaM*5WVnBt>*t))ycV9cPnr!rsRq5=Ap8Dwj6Z_rka=Al8D) z)8F{OYZCY=w|N3LO7Y*5Btx8TFQSJnS)!$5D8|(^MP?J`5uD6M0FC2G^9aV36i1eq z_>ERUX6j{Iw262!s1qx+k$SSxmfC3ItA=3_;FRq`^kg26ZNj~_kp#-Inq93wy>v9m zPP%fFX$qG8sRQy(v}NhQPE3ExsoqA0aST(T4f+}vU#?u-*mP`-ZvJv>XQupjOg#^B z#RO>PgOdqKO8m|mh)&d>dP(aGtUlvq%mp#-C6YB0S&&T53JHx&i7LlLi4Q8D*X=WXIe#|)&+o^_qoa?yozJI} z^X1*QGr%tJ=^Wwb{q4y6`TQn9L{9$azK#TEW8t@lo~#Jx0F^HSa(vNv#{Q12KD)TT za;|B1kbr=s!sR!K?bg*&pMmOk!SfgK#;%9^RM~+LmLRU7zSu_kT>xgJn}DR!9Q<8t zh(hTXS0x8dFm|u4Cz(2k=ZX!Z2np0A2@-z2+su3FZ z)#s?S7>ON3)T)cQs-Wed>wSXc6X{Fo0UPy|Nk%N+#;9XEXCXrF5GI0KUCC6Z!Vpm$POPn3Tfh@FH{&=O$c;m^(@fg9l>GeJMj`bh`)H?b4u^(Q z1u=1O2(@&^;;y~yHPXZ%Us5j3>B~7=rUHA9mMfB!IA2GT$Vg3d{|Kvj*IjEw^b0rT z_<=uqE%a`3GBqEnHV7=QCKV-SOG>~{?V<(=w`^fq?Qx#fui~iTuX|r2%f~p*P@zx! zTwF_fO;x)R-un(4%kIyTzFjgRu1%Y=nnD_)O(iq}4;o1ot(2m-Agq>eU{_424bQIt zi<({*$b|mxs~oC))a=54=~9vLz39-%rMg)_6PwPn#I#RCap8?r>WWvUlKc{@%o6?-FFflz zdV!5f5JjKS<#xQG+isqRXU>_eodfCJxQQCi&oH9#^PN^9Kchv1ck#as8a2Cs+u1WM z8*pyT9#L)Nq&K(B3hODVh(G!rK2~`(SanLdQ!#NihOadA1?g6`H~myA3pe~>^~?`%DyP}qzCGadOqs%# zx}2+g$jr}%swS;P!1pYHUS&}93%2s;N=>CR_v6KpEtkF%P!hx^JlfMcv)fxv8_hhB zIq63GE|43pq#TKqcCJ2kvg2OeJEG~tY{1=jvvvvSiO{z!Zshp=QojAXS>lndVx%O^ zw!gF1R*taSzzP8-=B~*+u!?w;qGr@Sd$Y;W-$2+)>)g%e?V$ zTP-q6qd1yI!o?=PN0lBdP>DCrmIy)8OA{$GX+qNU z)*fJQugVTlo&^i{${T0eaF%?w8xZ7g0(|`bN6=jnr3aR)7So1ou#?GWkf!ppM$Jhx0oTVcw>4g5qru&?FP@FL zO~*Qoi*Rd-UUlY>+H#y7lY2ejei=sz4zlCGC6?0QkH>wJYM5w{fp=Klsymn4RV7RW zn_b>tP}a!oZmLb!KBdX|o&;bssgQiqba_b7FBpms82jF~iXByZ(Q0>J6!heyd*|Po zxAyMB5j@70T22(R;HuQ2*kdr%(og6oEM2=!N-Gw;zA05xzCcmCSf94yv*b7-uBFfF zXmW|5W=1UpzSgAdR6b4U*Rwo^N<3ar!Sm{wSdo&C(VHv5{dE$TRhoVrEHEa)3ODIE z+JkU0(sY3(pwIdQL96;qU0*dCkR|%t;^SaI^>Q%JH!0ZX#VIj)sVZllvAejTH}{Nu zZz_fnJU{@XN;}tg^8Lc*p>nNt1-+pRuoryL&r*z(F?{KXjJ8wRw+CVkJ}jX@xZWN< z;td$MS}ofDU9%V+pGl-}SC4%VNj~q|dZXS5XFljCcP;I5WgT_c<1v<_B}v!4Pz>5F zPbS_U5bCf0(?7Y4YfOb}-1UMGobwu6fV=GcD{Fn#U4j3?K~6U}t6{x^xxH3s{hPP8 zuuTpu_bger)c8|`d{sui6`4B?t%Z<#dtbtdqheYtaB2IWcUJQr$7>gFeb~*dC&l1o z96M1fU~9p#`p|u&2co+KNi^pzXfE4uXZSI%fesnIiYUoluC*$>dUape1)q_-WJnV6&p7dodCnRuDPT@zq1(u_KxGg zyXZ-7f{ZBMbX)vcl9xqS=joSBUOr;;tr@77h3E;i-Dyxnxm5)}55k8{z|V%}{zrvW zqb*$`jQHkw^!2#R)oXEVdG)35u~8gMSIksb>})H3pvc)w1O9kcEMeE`5M~`X?u17S zXFB61r?5GK-i_F(Pa*_$unZi6byQ|J0Voiv~wBH;A$h|wT(C;MoI=MW~6 zH)DL@@&KlB1?ew!8As;)$nl@xrce?l#OG8Qq%=z|roQG0+$U-F!zqGj#?m+@6I^z{ zW2QNror*&D++8|+u@@N{B2T@W%)yG~T4;bh`oBW4yqxQci&)fUKU)afD-t}nLcVvo zmwoPSbCxp(;E41OXOBl?NR4H*V?U383~E#QFRXswtM9J%PEyGULg#cdlqezhTboA* z?+Po+`Ei%#%C$!79+Ej;ky4QxRUWN@duS?K z*IEB+O*jc`Te)N2cd3A#KZpMo1kG0|3d6DSeCw^agYamY2Jr5NSg(UQpnq+s$M>u& zSQM!i6he`0nbv1q3)Y_KKFOCQ0WdZ18qrFgPujO{7lOx8ODGyqREw0xD0G^n7?3sm zQV3bbM2g~T`XNGvUSjGzMpGYiqdG+ovEDca84&na^wV#azMfk~ddNxg5KT{5 zOeOv=+eiiji>3;_x+Ir}i62p~{yb#7;a!v|U)+s{p5V1oFQHp+Mx$c5ysx*jft7V> z6NQ`%yeyZllCe2ia9UDClSvbVJmGn`gO@Ay$c@R4Jwy7=p}0O4?EUF7UAQl`asq%Le%Iad$y?op!Z zxk!#FOW#8tFyU8WCsBo+RHihUPud7i~w=U^gCNVM+8m}^u zTo6?E#y;zbxtcHs9H~brBzXE6nS5hd89~s(CM6yTjnE*IpwgpCG#WKJO}d&om}MU= z%8H24BH_RG1fnV&(aQtTpy2{aE6ktLLF8aVE_nCSm^k8-*P5j(pDtv62byAKg*w?y zKKC=MT0FhRP0pfspZ4v2v(P|Cf~l*2J19~pVyjn}c^#ToKh{QWsfEn*!Otx<VBmY-s9N9Npf=%36x!{dGj<)u42pO!eOK4u;sSvNNW(vYn%lVTeAwUBHL^fJ@<=?g9C2({=?xjt~2f z#G_q!!hPgZ#@s~Yo0^dk_zJlE}|%}UlVl1Zck^|uSYg!@D`d+Z|N>hr~n8;6XN`29g1;7mMD(XG*V-+7Y4pEtV= zH!8PTjw%r%K?t5LIbrhB4&qydWth1Wis*(fzwYTWVBgRcdA10N9 zxza_Mgq9yip2)GxJ)eYy+7hbub!=c(BmiF3nx%Xc5(3bL z?ehQ#+~=d^s5F1?;|sbO<&I+aOBr5;wNNC*((3F^uV208HTfQth!ymT;MF^>Z}(~H zve#?xTn^7rv%1D#>SJ4eHQxh=&jR9Cf5vP`4-pCK^6uxo0+uqz9izBu#|V-g;6l}f zr-`t%-KbnR-VsO3g&niN9Yy(1C!$Os$a@zVnON5hkRS@vt-u+M_|NS0^!dMNk7S;9%3t^jdVDMW@*Z3zAFZfen$Kt>Tw2+sUk6G-uohB|H>g z5;_EbT5k3a8&L$TVFo(u6;dwEpc!OeF-nyuS|SAwAC#ep+O-lW@&G2vTE{1unVMY( ztew$sIv&uft$z9E`l~NqVufztSRjFkXjNyE+lKiR4yz;CP-Gr6=C=yJSu8F)eh{f9 zUDb^6R}Pi=sy98IM@zN&qDXZ%$+-?mnA*jICb=1qrV^$Kyo&m*x>kgWG4xdDek3?` zB2k^T;yf{DF-RUIVV8ze>&U57%Ae+Us*FXhT>aJ6gPF!$zSy<*7@9Q=uYAV~C_Wk9twlN{EZRw;3Kh8hJ|iR5=miNY<;7tcvt&Kx93U+*!Mi z270x-;$G*b&0hZN#XDhGGturc77g7o zpBoa2A~Q8(JPwz`5Xr+88v!#ci-MiHyXJO@acw9kI`;iGXJv`ybAG#(S}ZZ|9U25Y zz6K3>Nm;FO)uO+(Io^dh{3AW)jX+w8;AWq0?-&YnMWZq2(xHJo+`|XW6zg9%m9pL) zkLs?-G8(n{WHp!0`WHC)vWjT4E(!ed+5(A`PHhZg@vFRl#?m=)K5}*+J50&R@TXKd32!=ZdWiUNI{kl2fqI(#H-PZ{nBL!2c%hU;e39$%-veE(A4ZQe%|f z60+IYgo0|)Kca6#opnl(r&g==KsXy9EN~UA$=!JhXFDp^V$w^Dp4A|qh*XOZqXW;8 zN-q0Io;~+bZ&)gr+r3hwfHwE2%K_sg-)wiyDC7WiOrhH?eG6IyQNA-%>Bm}r*Gia^&!jK#MDhxjmUcM z|FdiVeSG(B?@uPJrFPFlrTn^Ej8svx-ced*LvZ79e>9QA|8*DldINmDNiz5E_s`?| z`drWZer1sVxqZva<@J3(I!gll98WCr`+gri_!G%L-`&rtPDQ5v+#kWxcq(~ZC!yy6 zc>DMVroKODI{d8Zgvfl;3suWgtCu=<4G5{LSNH_~qANc8x&N5Fmzgh8losmk@+K9_ z0G_}O-;6TD6Jbj>s{KxDyLV;ClXh8ZqkzGGvVF@n5Db%AwRRVGx9)g+<@A;sQL5ZC za|Fo|By#&0wbG3oSOAb*;H-Vpa~xR>PK^d{Dw6I^ndVe9f$g48UrA?+fKvHAq zV|-PaC89}7lCKN8#ueAFR=PTDqhWm9|I)#0*(Ti}a^M!w{(%0of*mzz7=M#bZ5>%3 zw-{l=HI=%gat7D;HM`rDhFVa|;ePy!<1sE`w5VqriJVowpi=iW?Fa#tsE3QodP~YFA{yu#8V3xq4#0OHfk-h;PcaWQZE@vPl7 zW*Iq(nz=sEU`puqh#NHlMjdk<_bw*Rm-xcjQH?$B2`z94_>sKMljZ+de4KD$9^1D6 zo`trCY4(EkoIxr2o6P2L(u+?#SIy?uWw0D?*-(LHV1wRfFFJWt!zli!o%&hJ6Xy#g zFnZRi_$c7)2A$h4LSz&k7mnubU=9A@I)ArFC_jcon9KOL@8M0R z#0=Ih%-|6XrRT1>Fp{3~;qv{wJvW<= zc}KkW2p3QhQdgpIx>cQmJLjT#~r(EUJ)H z+o{U1eKx&m$J7L6^i41<=1UQdlP1IuSy@&}{w#Y12`$=OVQz!pL%Lr0{j!Qwov*ys zFEIqeSNU#^W}1u@kf15*8ef=5<0=7R@F?g5mvZDLMkcRvXh(#7r&eyq({;7)<4Ce< z>BjlX*L4t49P+@}{VFN8~yh$BF$JKD& z5HMnQh>?}*mag$}YVq+J+??*xtiZG(c;WSGWHSy0aBJx6%=o$pk~ORfrJ-<=M4?^} z1||w#PV-Kt-Nf}OY(0G?fEE-X3_`4^JT3UE?p}ifF z@!jIWw(SmIJl5cQF9gDL=`^vlTJZdg@z-(s7$FGWVtVo4W=Q(ZF6QfUcY`6tkreEQ z@ut2sZ*@T%KeE#};eW?p(Rvk2QH%abS0<6bw!e-sMfxY=JgQ`@dmtC{ZxpQjD*y+f zOfZo}K4NBPUbGJP;$f9s1$N}8&;+(d<6Y;fRxnfd5EWoLV$qqzg0t5Rz28~HX@}N^ z=`Q9crQ>#k%VyOrwmP$es=lqAbLntbX-JPfAUNhM4;B96|6q9J4#5hH&{k%GIY6K` zG&lpz`oyxB7~*n>d**588zkWdYK~00yVkbb;JSyEqr<6#KB*Kdbv4LfO`uE^{PU|P zi5{9f;CxHGk1nw?)LoSv)Tm^xS*Pl3YyQn!BfWiNXsk%#Ffq)u)M2umA@-uq8yY#O z1Tdhy7|sOC^L-VN67~&)>_F?FB=RD!HJ`QWcFpY}Ly|5p zNRWYugkBN+$Rl9c8O$Ulp7NN?thUtAj&Cowdn&tt?D7~=mr-+39hNfyR%oNLY)>2X zH;5sL3}GctDh>HHA%?`uZqMdq{1)loLCoW_?RBcSAzx}A!NdBcUdo*|l+zq&7;D=g zc;y2VkE}=l_=O*%00w{`M<9?p0~H7qYsKC|DTT=`PDN2hCG67K*fXSnU5Z4 zUdcxf&)3vhpBKeYR$MLARSrByq^=tdz;-P-i?DPcf#w3v_q>_1v1Mc}q#a(;%qEuB zZJ>B{5((tXih*L=M3-4?H9&5aQ}-@7`Lf^$$sDO#KPJ#7QB}^QnumnNxa576qg>Wp z0Yd|p;iTTHsuxkG_{TZN-=#6zi%g$1no9Jej zTs|QQNOwusJ=!&?wyVtty;n5CBT!o})Lv7mLkG6+&P?cA?D$^xmVS*h3-)+>Q((gb zwHv-%)CFZ+^bze;6pnW;3iD_BG6tz<{KKm01?sGddDJ0QL{T0Wm-T>HehIv$gDAX2N*7z9~DGBvN+z8 z>HGxk*|l1;BR2G{+HHg+e*5b%b`O$bAY9OAjF4`Yu%u#V9`d+@#fba})wf*g99DxR zjcyQ@BB5_s_;GOV(_`eG5oz~CB%l+xR_oShf%cnXCYhZQEf~joeRR!m?B59DF3@Uk z9XfHYuu?kIGX)dZYVjW2nxfU}FAXK8hWLKbg?VP-g8^!K!+9WMxSXuT$i3j-nv>2@ z_}7gpwAhO{VF%Lws@aCc$!fOcbETsEKMjaFYKD)tH3!t0-7M(kMAV~L7)sK{VOCa* zp-@w?+@->UMrn4 z3^C^abaeXKyGgN>y^44iC6T4NV7kunHBynOk<-M;3*l}i#Q48*<@`F+Kl2E_Yx#LH zK0grsf41YDENd;q^)S^lG>o?es587<)_1Ivx|bYo5+}I8UFGbJ=>^9sdE8XGJ><%# z{AK@uI=u3z(x5B;wr6GNfiKwK{C794$~mj!dsJORb5tO^xWJ;&leU^Zk)%q49SUyW1BMvq$qTe(Sxg5#9KFeXx* zA^wR>JP0uNoGz9AV_k&C9uZz)b`nHg>*HiCB8)sMVSTH@>zt(J3PLU3Lar`N;QHDnya;+ ze8pTo)b=^rzKHSsKH0WrwRSGuF9%132R%N=y^R@-1ctlaXY;_jEprp^8>ksK!fx&H zW>C0^^r&$Ir0BXgy|z-nH6TCO8W|H1_j((|A(+5gq_EA9U|b$zm2IqQPWYEmdKfo? z=w$Xv7BKP9Y6lE&#hWq;A5T%2C^dnZb_{H(oALlSw8H3V5YJ7h>mZ7n9{dA4R!SHN zx*?x-luXERPum_h_w5^>XR|B`YrMmj& z&wYTn6x-0>Iz;R0>!;84MgO0}`3fzh->yf=g?hB?bn8P!=&1E)SYb{kE_?H~Sm+qx zcmBDOY>t6;$u*+xEWP6|oa6W8S)p^CC^TAk!Dj2X6=3Nea?Gq|OWC|gn)5s*roUO| z!YgxU+`&thUW0&aCdr?|)3L0<7-@1>Ro-&Qs5fB*m8y5bsC_R({KJ>f5}UX_h%9eu z2narK9sdVz=P~0^W7>}K`0=Q398uc;@b2~B@KWc0&a74A@nhrh5@727Tj(!V#}UbS z>IlU42Aa_m2)sU3u15c(y+5A6hXSE}Kr!b+aaKG>i%G2&t&KsLTakTU5cwgZi45c{ zZ8(u5dbBw4Nu$g;o^2fteBQSqY@l}t!d|7?>*>}w?Or1zxE8J}?a5vA8l0G;xR5)J zBHcOHzxkTBq7yg`m@nzVIv3!VtOJKy!{U|`Pi%kZx7K7dU-XmSZ|%!r)$59OrA~uq zKEP7gu%})LLd*1id+~MuH4gregyTPFzDl#J_|0u;0bJD=G)x+hTx8`jvzc&}O}19s zc^EE?OiqCZV^XBj05gp*=G)hdkl$3g;jNcG(%5#w6;ZY781ek?`lP|>+}*8?I>`bo zW}}I{u1<0)9CKI-PivZS{*X5$4Qk^_Xg)9yWVR^|-qZeHMmjZw>trJy!?X^rI>`5jx3#+w0PMJ{%3R^9H-Tb5)Q%ZRpk z`5MvZ-t=7sl>Pki0Mzk669dm=(Tz~gr+qv&y?QftHB*b-d(e~nWo9EzW8bK!a*e4x zg@|;jmnA<}I&?XkIBQBz*(9f$n{0#DaE6b2puW6KocvhZw(a<5?DNjiN~P|dizcH`U5ZM@igmt-^A2>e)h zvvRl3`Bj{QM4SD9ApVYcoHt*{qp5_+iWSwQq-}}a zG2e6O{ylhd+I%|7JdZ6Gh6s3tar(SL@dr^%r0wFtmeW;VkrB_!G%GKh*0UQsEh%E} zd<~aolGp&yJ|6$KY6jhYoQ$pX^A2+^vyJ6vddK>VoaVI4VU;z`z6EFU9&WC_i`vsSQs9Wbe^gO_V?pc|dn`!O5)=_%ei zZYHf^_@nLuu4`hlm4Sr(@dMKU5L#gx&*Pte+9gEKk<<_HfbNM^Md+C}&yiT)2~vL7 z+j)Aaux%8@KW1Tijshu5#-&aB#FWy#jSCh2Ov%}@4SKRgFYQk^^CROdIg;jin_0R& z`(Bpq@^1N!&X@8xoxRME4Cpp(MHXQSy#jAd1e273=pWFy!N_n(!H!brUez7|3eC$| z0XtQjuX)HSH@^W)?`Ia#vRDvz>rB*i89cB4zK-$t&-@Wq{Heb}Y(>1XODw5rR5jwy zh5gz}q3a$teTqchpqBLpVWCwF7r5C1cru`^{TSA(=_pZyhH^p){&_Ff20;#+y%@UC z?==w*bEV0ZN}3N+lNtK9?cdp-e^-Hc23^Qjt^k6twQ!qM>EuvN@77vFxAGKLvPS6> zX0sw_b($`QyW*qnJP3CeoIwcUYW_3d-?X-tbO)N$^tVB4yPt-*FL)CuzsclL69#`| zod(QBb42gU}vcv>cIe5*_-cQ-`P(^^4FBodHH1_Dn3t{>A`E}naN;d-@~f+^AONh zKpd*gIfBg*CK97YLYvBn&+2?lDF_hvJiRd+!VXyL{kSNl&-DCQ_|U#|+m402u)G$p58bCO;8LgSeKit?Gb$^D z+eC<$rmu>>iP^UhmdWYH7=J?xL|%Sct$)XfoEyX5Rm65!OM|aKt!@U?p!48t9lihS zXXo9;pRC}w)UAdMQf--uN+Bfz8zm3SgWgKOw>0HcMRaKA5hBrk#m+^3|A|?Ww09cF17O}WYJq_Dt=f%F>_Flp3MF-5Q zZFawKso!<60`{1u&JJNpnjNf@nU1Qs0-Ka-(`qUcjsA?uaa9|xXr7TRe@j~y$~;6H z%QO<9`9XRXoeJ66u3^rX8ozE+Z7#>v0iIYtjquaUn0Y7_qWmtF^rb1-PXtv`+Nv*? zE6;3~Q?hTpyOT>6At>>qfC~VzW}aq`#h$=}39&>a0Q{vQwWc1mheQ`%sak9dH;1*b zS2^unMKPvo83Jw~Iy?;_>#AF77K$H=Vash;womy0xiH?WeBN5hA>GyDCt>muh zYeYdFLhooIIX~X%3k&xktGfBIM)h*;N2Y}DWyru`q+BV%o8e&CclEdcXs=-= z@?dj;!=sDP;MsNEL`MQ-mjh)`^qH;=wOUD0p}?QmBLhaA1ayonDlTD~AS@H!exMDm zBgODeaA=>)DDn+{K7w)_Y|`mQwN;T#u8{eW@EBMP046(0Ss8uP0d(@*+KrxeGZBY_ zki=dBn!gK(VDZVy_|$bR3T40g40hYyOQf5M+$G9wCxI~xy;(@$pmymU7mZi5RZO1| z>O-4lGQMi9|C2om)l<%IAIUMUMS!bkRxv%k6;AP+xT^=!(Gp_Ft0|r2+~kJa#nkdi z?+oH&^T`zn1Vp#ZEF+&__eTcMBT2jV8(c#X>UOqAGR19S^jI?k@ul{8`R#*Iz@sZxppf8~& z=%ptB8hsEEgAdUaW4)tJLIoIq`$^mxa=Ak>>GJ%-@RWtGF>H6Z4GdU`pqj~aS2pGD#z z!>#rY#14-s2G8oiAh#MLZVEDQz!Phz`Wu+o#~8xlS;kwSMue}MzGe@-s%ymIIaF_s zC3kK1S<^eCL%ASnZG9I%Wwa70#P3Sz=SV1Lj+K{FTfg+vx({5Oq26TE^M1}c6rv-5 zW}9i{b7$4=_5b1-B%f>OueYT_(zf_;5${Ob9`bu(!cbKm!?&?r9&XGs>?gpNa(S~c zCvX-yUe2+q4V^k4PTBkQXRlKpa(P^4%AukfasKLP6g+t%a;vQ!Mf0C^uGrEUj*{Zv zT0XUQuGBs@H5Jop-E*T5>I^;+_{Fg9&X>CEHsS5=1SB80V1qI#&pv2sBoJ!|NPW&( zxq*@H`mnWjI?oZEDMosFGgty7WURvjY~~-Bg98PuVgC@wYvy?#or;*D&65{XiUq_{$O&F|Z#tFl)wv%}?Rvu}9b&3}a zHkA6sZ%XH1?l*aGjHjvA4s{9~mLw#Ll4~c+^bhic=}V#MXGN6uF%?=S zZl;ko9B5LLEE+h&VnDpV%uMR#WoXKj1En+-Fw43}WxlCZZjUf((mfckfB>4VwZB6- zH9*WQtv0BmD>=oYjy-#J$F{wcWA4*mJDy)YWr*)u_iTI^mt)$jhpw{j-nHA`+IMd3 z2R-r_J7S1Cv*z#MAUs#@zBBIN(5fOtBT)-5wRL$Zv{l@x%U}JI=kMCJ>=jYAraP|hs>_Mk#N6zf815E8nCxbK zyA#-UwsQj6OaULs-mLS9u&S{(akt{ad-!Zjy_#}OmWbaN(NRq{%Bco-3^VURVUg#p z)XKr(#(6e>snoefv7;t6Ei9LeAFIfuScF^z-MU%340yQPkYExeuP^C+8z5@h(0UgS zq6lFnSr<22>ZLLQ-M6)kyPTVP=hb(Db&?59X4=;_+CMs#|Kc3Ty=`EYT`~x^xTrB^ zo3_A0PXo0I{$3F6nS~Wftc63i2DA}of|1t#S)v|YMzIqIZ;)(>v{0<_v-=&XE}3#$ z*E+KL`){GpO#qwVK6Q_xUf?;~`ca_9MFYigKJ}TcM3OmPih8|_f&z?1BH4JU(k~1b z!Ill1vv9<$MKt=b-gkN}=w_L!)JoNFX>_mqD9G0+-7X)<&+;Vi{sj)lthG9S<0hZ~Ep$?>93vp47>W4;mbNz-$4VhxyLBGZcTn5CI;m~J z9M`G<-kdcmfjkqK%fh)n(dR_+RuWI~6x_vL<9QcKyg-0wMegsTUKK3QH!Nl_V`GlDPWkhiw?zPYGZNsv+>xwJGkw5L=fU|8ajApN<;~zMeO8(?+4C4 zLDp4D4WUQx!w!$$5Nln?JqZK2gzRp!kiTj<$ki=gJBgjbxYF1t!zg1b$IlQtMD}fI z0EB#HlrSvW&PQU&y}i;1N=T%Yrc|OpVo?FS%$lRoYfWi8xKmsL<23=tbs4+M#AP&{ z#2K#*vM#FJn%rS?&X7Pi6Tu#RBV62QlzZ8`qdO z4G4ehZ7Mkzgkl7Jcmm=CqOvUv!G-`-K&!v6R4g~ed#Q+ud&#UM;5lC>#BQu4nt4E*fockwu|amoOf*tp#5=>Mo_uyF|( zF;Ek6&|oIXM4|W#?#v&M))*MaBdnCYe@*DT=SHwwba6|_8&lkk5w*;{vJL&yb|z$~ z=jozKwGC#kVNyAxnfoyr{%EX2F_z6;Q`}ZJyDzPsu?^Lk9-#E-94Up@b|NW#u`RCO zhThN=3WhgHJ8cFPXQ;&@h6w8azm)D5h&2uiO)QEO-})y}YalWZQrY}dU+3*}baQii zPUHVs|9p{t*lilq@AoI*=i>JM1d986JdULKdw!nIuNO5j{5U+nUQUVU{eb3NS;p)r zN`#CM%+7}AOI~}*QS;<^fBTt^)AYF*?NB6593@4v@z1+tcB=*pat%IS*Y(q{ z#PQ9ue877BN@>wUJv{R-r6LkB@oyylWz8z0FHtUT=QJ_>6|CgMGZmgr3zKDq%}QgD ziUQidUAAhzD4w^;RcOb%Xx450R=pk}PunD-ZGEv7B+n8trwrzPgu3(Hz3i^*?<-exlw}^$zy8bwSD@uoVD)r2jphKWrvxSV zOS{hB=!1YqkU=+vXQoAGkq6#(A-tkY2Kg@8d?YeucOx@W%?_GHmpwl|y*Uy=oz!Dq#u2Tp&K<^3CP~aMA^s?uxlt5gzO&_ybXf zvulp77LuKcf4!>9>`@cw(c1^lus@xh@Of8N^y6KrFZA=_Uo^BTBJIFJa<+d5c-~(r z+XDak!2RiPZmZ6%OYx6;Pba}#6hX>IXLk`T9<1tyb(x1q>ug-F`S&g^TaKE_Lu9xr z?}+6TIv_j^Y1I8&cLpK74Ev$I@`oA&ZEHqV^v*T|(B0}6+~gB6W2_L+_h|c&UH?!u zMZni*_u0A4SRl!0rE0dBhFC%hc!dfQmD1QgSBlwT7pW<=DxfgG@#0;d*fC4oc*49I zI&#G3X1{ck?mu*vK!x_SiL4d8PR?vb*Q|US%7lRgwirCFqO?{ssiCbmqJVT5t%>F3 zu6fY=3O=bT3|QUj1XzWcHRn&>b^LGPrj45YH4?%jJCSKpmPk*oK`LD-QTR(x&QXry zudCUhp4wR%!qbsq7+e?G8`#usc7Pc+n`U3CzK(74VzIuy&aR3^9b!?>NBvDp_02iV zW2hoqSe3B#a*gh-n5?QjXv&C9NelVr6%^C<{9+@PWWq&PrrbQ4u>4)5iG0JQ9RDT| z^fV_GwWq;RTp>++1JVBfG3pP|nSNgvC>qyH2L zyQgt_r}L~i7v{4Yd(6ffn}$rm?}D%10v@|Mjn9FDOtz;?uVer9EOo&CcAI*=rhVzr z##h5}sr&fn4}}7=EL8*s=WE0Mn~F5*c@dp_ZQ9JT*Os}7bP-4~ofb^2gmIJndETE{ zA0lMB%JYS+%&3adX=~O&mF=!*IV{yERo&x!;S~yb)u9FcVfa5GIgl(IJ6E%qGn-{b z^0GN3$^=SrwW39c%9I&xI@uBVrj_q#>F7F3J-ES48LOb3G3RpZ38HE;)Jh;)ZJ-ya zIG?G;zPi&FAYBEvORLtKc64H}=_5;S?I;@|KuS1*Kk~OAI)NKAxGp+zY825f-8!W# zbka@ljrp%R=9b`9diNaiptf3_a<56~bcpO>Ak^4Gg0xR%iV zXER2j^)P&d_bn9^sk+wh0&+i(Y_NLe^dg|^dXCd!%{BWmhPy(-n(SFxp$lr0Zk*9T#9TRUWTW~Cmc!cLN=~5*3-4s`b&p^BZ5U_d?LSmLt{qN{VdC-XNUaAc zoaIWx1yICmyw%zAnHe{YR?9Q?Y2nE((jJlS}JBO^w@Fjd`_CkvF8i<_GAf`GRE(vHJvY#nSI(Zv<&?@=K3VVPst^9hA%e(-weOviY z3#It|uqcYrt`sKScPCdEW4cT}F)xK`arKcI&V~Y88h`|@q%C=2ow?4S`OgGSG`~?T z&v{7+n7!U5KeOQrj`HNw8-(?XHnl&k&uDX;Mgc^dB>iA)_-yjrwHbJ;_0C`pS}F~7 zFti0aa;{Q^keO?&al0L7BUatnZ4V;>2ooNWE37`>tf;jmG6zZzDn!gxU9rM>B4lfva>7~*TMb7=#22rkOyq5A3 z;`~TU$!Q-V(SxY_&4|evitbF=WgJh#0j-giwxc@JC4@1Jx9FfKMlg*9`{KOayWmw3 z5*={6Q!V3TL(e5|MSK84LQaf3m2sc%s221TulIbpE)owCb#;zQ55R?HR_t*rU~`n=7D)FFnyBEQlnLe177V~6ddk4Yo-3)5xAVLVsW@v$2+)RwShZm!GBka+%sZ}@>NiE2@<7&&ENO~W|;y#?8X=c_FJ02Qew@v zhu8$SN|YbOtlGfYkHLhm*el&)3@O82%H4#`#1sbqI_%&kFEg@iDqrHIZ zb6$IG%b@;ZNWvALJ9rt-RW?>81B?&5x0?s%(3(!;;ylr>8@rATD})Gn8&2svQ;Tx{ zwVUr$nX5;Ho#|;aYsWu&R@{u2w*_?|>mLmUpm@#gq;STo%v{R=o+ zX|qRXQcR5;ffVS!#hikXvAbb1+aa~!X1mad?5#O_A5j!;xwHrWz4lAt&o(6My_YT0 z^A^ots<O}3qcgb;E*&mSZ@Ey1#3@K-IH`DUr@P=f(!`j2gRt=PX0s@57LgwVsh zhh6g@M#}<_gz*LlPO=AxwP$J&h>o|tHhDJ|>kr2&mOsZUmVL)6mJ@jWZTE^5gf5?o z74}(vcHKV~>sh9RPqEF3=dsNTCLrTj6A6kD*r$Yx5l(#n4krfh#Tt@X@Upa0wgdA( zr@uby__H3}%E>cx%k%NEO>-59XV+HU{$ytNjpHUhuSxZ=roW;;(OI5AxWE&iCf7&F zuTFXUkSF+>436Vx-wF`AT8-gnci?AJyhzIh4c3u|PhM;&R8}fu$lw2H4!AGOwR$Ju zaNz1QsEEAyh7;B~=Q^vWJXLj4GMzKyHcmDws3sTXMn#KA#GkFagge$v+=MvZnX>6+ zs}h{FRQaysLiFX`p`DzcS_``M!CN)P58j%ffHJ)G!;^AG)kjyfbT@6O%H{Bzd10rO zdByZm6K9H{0qB~pN50%)NIMVfaIa~O6EX0}ijg>|ua8W`ZV2qbSpappn>1}1g*bUH zl}JPvDE`}TQe;F#)cc-A8tbGH)`{G69vtG!kaU_G0URJFD6&{vlUv~tIz39CQcqPi z1&izfh+xsUWlebqMpl~YPFP~tHjZN6*Cx}noK9qyoGk&<8^Q&Md|PeqcEkpki_GP{ z31YZ5#9}vOO&-|(@9i_@;8=z@>phgs1a$*#jTOe6crcBo0Rm;s{_?BY^A2C2mioo} zFKXJUP$4^$KVcq?B89sTJ{=5jh;6?LML22LcNoJXw#ycf=%7WY>sH(Fo^57_JCBhp zyBIpNVDTzV!(HXwe-bbm55T?^`8~(gq-rIwyPXLl*j`oyu@E)pqhv}RtmD{*Qsewv za2xaOb$@{6n2zasXxk0qwzFzQp?pm|+}6vbR#DIy*8?3G?N&-O!y-Ai;BUH)Bz-rF z6ky2c_MT01l0tpmy8w>5IZrdTo$2qQhpvk)s>l@&ZQznicopO~)>qXBaD_CvcpU+gm z=OeJH1-^TN^>o|qFtTp&tQ=y$K)&AEw(7Wv2A4dL*CWN)J=-zKHygK|QG69j>`ab0 zRYK!2yq-M>jms{atx>Z`2N;xcRpyY=vIL5pk!oMWcPdjISWV+NyBuWBIwp=dS8^N6 zk`y``E!?X=XF4;p&^lsrXvT55Ec6(*HDy!UpcLd5y(zu%XYB`NrnZ(o>O|RQp)NYS zW&AOgQV&Z@LGn&z^_s(Vrf@vp!VYn;-E$s$%{WvQwEs8-s~1$nP6 zTFD)hanV0+S==hyCDUM!UO&5pYkki!G>u4HA(P-|>2_odYLTC8nANo`nL3hMOkykh086Vj=0oEIVV0nx zNTrqIKDWkC=BNte2nn2~PiAE)8j9y0`m~T1H|&4DlC0OZh&EgjjpBFyCk=7r9bXH$++ zzzZdh2NVv+j5r@nw^<#9?^3koipRhk43tu%l2Crtjy?V?!*hrV2g%b)K%6R4?4Tj= z&yQsl#a1Ib)I>K-CiYxW|+SZ`96wf!TivZN&TX0`p z6ng)uh!94uKM^Laz%uZMDb5)L#z8Y$#iMaDa?i0d|yA1n-8ogKB*n&~hWj z$`R#?KolOJw!J~a{t8S~dJ-|dC^&==L7KvjISO6e6m8u_1dEI2y93f7i{8faypE+n z7ru<-_?k`*!#Qi#c>A3xq6k7{-3eS-YMprD00_gn>nk3lM65vg;;^7^Q7Htuo6kj> zdK&rPWcb+!4)0N*ebR+^j&C>EM+3e@OSFm5?(5!TiyR#dgmi%Y~K`=GT1xp9xo zS=^ybPBj^nFzlS*>*>@!lXx0t+0<@MQec6# z2e!~J&ZEM7BrSSkbz^E6CZyLfUy>YZycBfg)Z&_=PpGI9QPu4gipw2%_s3 zUBbK2?lT@V&%uZIC;oGvJ-)CRHXhn)c!;|MHXgd{jL#H3R~S9FPZf{v@agx;2CU_E zSo5&T;s)$E*VvIMfhoNpCngbIY(m`V_|ce_UyUVjbbRc{wDAr&I!-FYf=L7qY<$Zk z__sGYKBmA;q-h)8h(32$RzLJVsS2gIM)bLHsk(6m?tZEm0goY^KOVloJf|@QZd0n< zMH~*^;u-A!`OH-R(0@$QQq2kYKNnKZIrz_lSdPU!B;)>{iAMCf|BlG!G4#Oy$afvK zJn{U;KJr-CEKV88D)Kn^!jGlUgS%JtbztxEfvtcl9@?&0cdj6xo3KE=Jy%4)xrgtn z>lEQ8`WSReE&*o+eH^^?I-Tuh1jqLXDBuicdL7M%*>R=6qIKXv@5F`KaYbMn+2&`32GxxHX~eRjX-?uz}N0Ke&Vbm&+ zsp8KQxXY8HI8YUNa+XJ5d~h=^@4qhpcD;0Ls{gv?lKjwjku7&BOt+F)Y2xdYhKH6G zEm2PC2UjSCNxJyDM9Y#6o-+zjs)txFkTM=2q={9hYSh#tgD)})_Iyh<3s)287g59U z4UjQc(SfPHlNPY-TiSuM)LvJ(atAEAopv1kIV==coL4R??dMs2Mas`ar7u99BQUm7 zU<+vwEw`mtkQh{;kcLqH+hQ`OI;HrGU?u95d>>M|{O53O{Tm9QcUy14&^eIw*D*cv z+!YhCbaF&{Q2>k(%M*s&rND)CUV6H0Xr%BW02Fx^K{oqBaW}s}x0#8{N7q4--MseS z8D4hM_8*aRuA#OZ@x)YSiAF^)N4p@B?loc1#ls6+1&irK+$vd>n7AO@RdFM>=gy1Y zDM5UUbNm2yX?u6C##lBpK)R;)#92sv8B8yANM|uyNW&R`-CgNcYQPX6E%VHXthIC~ z8)x7lvLF<}%pt+VeG)bEj^3h1o1v(nyH0l+%`eX_D^+(=+4pHHvzOaD*o6K zU6}Y61L@~U^-a;p(7`~!_ZcFuyz*v6U5b^7bsNI6YVrzM+)Di%VplmmzaXT{(BANd z;3e){@*Tgyb3H=NRvC=4x_zOjCtWZV!c1GjI26D@s=<29Pez#In3?er2dn($iltuR zk-}jgLr(Xhd>{%@Zqya_y~=e1Q45#h|E$qW-t{7FP!?rwkJkSE$^d40kkmne-*HVT zm1euZ1Y(N8bWZSsDoH;X#wwjz5>p4;ko+5|Z;k)3JXD2UtZBlPv1I^S*x&7xDk2}} z^$8sjc4J%-dRLFk#uDs8^>00P`bVp;R&yASK1MA{kUz^RRQ;VvjR5oOxPvSGDIOz# zMw@i2<1TzV;OClg7AuevkwPa@2rpyG7L3 zq5O*|aIU4Kq7(@;+9ZRtqt=SI+<_7Y#Nx|mFwJwKzLo!=84Scx<0-4piHZgpf0RGx-@7b)X zxI!oti5|+K?7`VEzba?QDfE6!Dl2UBN60zo`;7A2yzH7Vl^Q2oU$GxFdQ5Rh()%N_ zyCTwC-qA7@CAiNhC@-2}u=%wxLFPi}8{#+dAcR;+s||(mXenw*xhkb?DH+sh=O(q9 zmarlUN(=&C08&;A()R|({cr`mSjkFT*Q$i5AU>&c;4a)vnPFkl_1AVv9>Wd2i9o+^ zHae)_Ab1axrWS*KREGNupQi&u$iNa^eGq4AgamP3zI;tdmhzAqS%a{%S?Fb4LzauY zBf6JUlOfY1wNgA-spHoi_2>W4DYa6(nI#6bkY6@BnG#x|2L?@w3_Y83o^DgAVHK;& zxk0$h6uC5}D>9&}S*G8HMTL>?2aWKuVWX{SgPl)MwSf{UDJp=d;rmfv=;U*!%dwsh< z3q5mst=?an!bCD{z~piEnTOlgF+Bt;_VhUx32k))X4ya(;{-xK$1e&iS9x?4w(F?! zo2x`i4_1)uJ2Dc|I-RJAOORysItWa51ko1M#qNXF?VRaPB+_FtD1nCGSewK^Ef0~5 zxRy^+ z66C)2&8XLFd*%~Ka`Zqajjd=&c9Ct-0IYvCm)TYVb$cF>NF**7(juPJpJ~XaW@hCn znZpx@D#zqR#`6T$N|-g|zZ^H$nkxz$%s)Qgx@?16o!CB}+w0p~^WmyPJ{$GFSG*ZF zf#o^2A9UUdnk{0K)_T?TSMxe>^vVh&Bl*ad6HkYU5FljIfj0=l0i#WE=I5!ajb`>8 zY2pdxhF{)37HH~|IgI6g@oA(X0wTZuxxd_Aj5kjPetnG-l?xFCLjL<&UR?CNGwAtz z*`4i<_0$}Hsy?po6a1;MwB)VSx9&wy&RMEkt~ycuCr}Nh;O9N9)m9(J(}JF}L!GfP zl3epd5Tcb_o3jRr*+p3ZpG$8mNiDF>UYot9Pgm1&?<9Y{DXI-7P+GSoV4Dx{V4>;a z@GX7c+OG<{?EgNmjs9DI1Br3XwtN=^56??M%TsH0VhyXB-*(ttTbr_i)Bs4@{V;V@ zY05xqHyc3N}=#3ORE#>-|{VhR@ue(0%RO2>CIopLeE zM?>8z4B6Az1g_VO+1AqGB&2^g^(n7hNlwB!mAk|rLuQySQV}DE4ZywxCWeZuuc-62F5n>s57VONMzxvfV zgutXX*rxLiOFx1B^vg~6@Uix1vEW-g5qO*$#W?oz%dI*+)0roUodOoK-AA&1O&kU; zQu~v4RW!UP#0+l)p5pU8Th0p!ZXQol$T-{@7TRRnZZB?9dsizV`9wc?I0B29^rULd zXyQoL7*~E4V$c(YZsH{3g1;93K~yAqu7vKV;T5i3CUujPLg{$0$TggdnMk0YYhwi; zRib^Vpd6gVkLOMf0@->=i{8imS|h(j3!@Ans@E}$rV&(mA#oKqqW(Sv-^b=c!2dz+ zH}8&b2T~a^#qks@=O2vkAV^4C1)6i2r_Dv=^ZGz<@1LWh{;A+Tq7(c0tTU>#Csl{5?Xy4q)Kn$6NULaj{s1rF1b zjnF582C-uq0WfmosMq0M*91jM<+O{=|_%P>a3Cj@SMVd%U~An*M=CzhV# zq@H+l7m$|JDFW$Isq8{DFsGz7^a>KrGd+m|RjF-?a2poQELbAj`hlR%x3mNrIVRP- z6pZ+@cJ<&9#8G2^J+HE!BR)eR;x#(dnG^ ze?30#?|&(!5Pd|TE5s2vF2+j?Xi>|cdWAp#kH!YjqDmlsSZ<3^994=iUD(sI0CArj zCF5ieilBN?0t3}T8r(6$y&RsWx72A~_tHI?iqrUg67aUio41kJ&%6fLgbpk~71wx- zv4AHmCt$6xOzYMFy+HD-y%Be zBNT22pQ@<|V2MxPvTmok7j>bgL|i^|-mj*=?WTGRD-;x7L}7*2d^e$XzYt_e zgV;&kUmMYYMlBv$#x;iZ@`!~gO&fp-6FfGm69zF7;(H}KFAl`M#vN3X^f6r&>vS1+ zHM}cDTxk?g4z?ZXZW@dH+gmAo9*Rn^rj0AuPnxx5CBltw(#NZ0hll`AiBR8)uWRz( zXO?I0d9u;lSy?*?={uuNG*G@YCaqZ*7S}GkkfLM%0v?goEi4$#Pn50_9EgR~FBE{# zvsb{l>Vck#tfqQ0QN<#>Ae4(m0!Ub`H(nG@`TdV-GFZHNN#Sh$O{*~8Tfg7^!uN;*PE*&uCu?|9e5ho%Q+N^6PO~7Vl*W(3-ci)-LE{nSGpcJ zfIB>ozDp%z{S=XFzJYC{atQ^ zThLm!C{^qUf}^jJIePV9j<@o~_$5uZ6{;~N4Ino08UCNjlA0?oWetnyXare?3MoBN zoo6q90NT!tJkCPNe$pB`fJ~`TNQ1VhRfIcx z)~ep5KNmG0R)ikbbIeV~(Vqpx*!wYp+$EHR|Iw&6v~uzdm%Z72n{w5Ho#Ms3~z zXV}C_X9V;@xg-fCL%Tk<0z-+&1xb5vQ3LAQmHAFBk;6ZrCrLfCoe-<*d!3k$UMq&` z4T4in>Hl~?{wUs2jvEpXYzn9UE~D%?eXBn9yp0qhAR;38e%p<8bgW|xC>y^EFdMy` z`+B;)_~r88N}hOpT;CRI(j^(ns!~R&CpWVu6fD(Uapk~imGSv^>meeBJALOGRLLKP z&ISF$i8~4QZaZU)5pNZEjN4iWx{I?{p56ajoUX;U34j|1&2sGNz;6XH`m4^q1%Kz2 zjU^fcHyT_}_)e45FGz6MrJQIX`Xi<|_go@EJ)ym1|9Hm(Q)4$bPq7@1k9o%vaPQOl zrGLRcjba{Q1a?qGn_asJj7xC3afK2c4JrE_B|{FtjA@d<5w5=raTisM(2yvzx{WrV zIc;l7K_6228)ky#WW1d&b()Uctoc?)WW^{YsxpKImIno|a#LCv=rOnk#VKiRytDq3 zK5@#0*FZGoU%j}%%V1-mcaUG|5(A!>geS7_`m~*%A61~#Jo-+jYnO6-H~EXzLgE#O z%tO~T6pcbr4ZiEi`4pNUj8ElMrLv|0N?oGqt)#I&@Y$Luvj8}DWusG6nzXle9y))HD9ugi0x(OY{fk-S z*6r(WW(mE|t(nP5#F zwpD$K0qx|O-#duo7h#&rjzV|i37lL_OZxbjbQS0r0}*zM&le7L563VFa=YF6wNO=! z)Pcpgkj16u5!3!ES*Hk^To!Y7o|iye?3!$s{4<3iKbM`UN2-3h$lc=xJIr~wQmwh5 z6JL77fx%#D>PF2Y%n@~H#glQVL#HE%wms5D_X-!%(@alBQ&bI*ZZ%1>N6DunMTxH} zhK}D5M+pUMJAOlJW+wxiWtpw{SW`j(pwdk_d6pS-0QX^Aq-rB6VX+dKO1VeOHiXo< zptK|))J4>%r)_kafmEJo0J+7PvWrK))O+=O&>t|LPc9Rz6{deYG!JG){q%}LwLcL3 z_YbF9i;+DP(op~|z+5|dW$RIpp53yvl5+<#$?_!%oeMERv1hX4tZ$IEQ`y-#khLR* zQ-W;j(gIJGQ)!sEA#FoR2C~I11hk z3Ng_wO$?ef8NZAws%pdGHJfx*$99W99i5y<1uR3l{$5wA7b~13S9u`;X`djUxG355 zDWX{na+^njtL$F)y%CWKg{%=%#lOW{61}Pkae-w7-dyExLz6f6#Sw!#Cn+Ty?eAi? z0dlcrlyvH1TbC|!ZdKW11j}6c0E&?DTl>N5ZI<|`35yx44i5f4-GCPAn#$ewLbRaT ztB=>5oNC8tnXC+1P#}FxjNQ`FT;S9Z0v2e+qeI^$s z;$Oug#?M&ooYg4GI24;K)VneOZGvW@3fMN37%$Sf&smPXz~tGFtqKLOEZRP zF|v>&e)m$|g7_e9G5mZh3Ts<` z9sAgc^OEKp;bt(MH=d>D52l6wy*~;m&axAb&n%q2Pd^&g1$x0vDzlXP+-=apIK~$t z$zM_RXw=8pog!6#jg1XhdzWWfmoECuzv0WOQ3erzCi9s~N)`C|`>Hb&tD zD~82iB0#NCL$Hakv7Uz%-~^M=&fOOhY(= z@%`T=*qcUC>%HuntGBU+D#`oLCU=F^%P)DjB)Amo5g`20m<)H-j28w4hIG#SZm-1x z+t)a1qxX{Z(k5fsa(8+MNg=BN8g7zoxH3HAW#r++D8;j0wcHIP*QNUVnQGVgq;3oW zmZ!};WjL9?;4P-0L>i=$Gc1O>SU$McUy%%XGSurYu_`6i%-YL7M#DP4=SpB@6Xhnp zspX%5`yum^0_pHBwH+Ba?K{Y4;e_X*rzPM$$bfDMYfw)Bn^d&R8o9wM90cj+Q=&wZ zn}?8O2|uKeSQ4H;!G$)-M6>~q_F$F5QPJ#kG7K9NN*eaDeH5ku5 z!7E8n(46h0`^&Lye{P%x2O}n@e8{;b`EXc9dKGtvP%I1o*ahW5qcNJv`!R8tIT~=; zh{SQ^YnP>Q*uY14O4%jfkpp{TZtcB!0ltrlDwdB5Nut$@VqAZ<;sl!~<<_zPux>Tp z{}NS#{}n%iYGnS8sE=d+{~O~3tIpo?q|u8LF3Db$A&H#h2`_lDGaB6?p**s+ek=7x zD#6yRu~vG5s&_(WoSO&;rPWumnNIwUM{_*$w08Z=et!0kmMn0#%drN;Ka3>eAbjTI zlkLxV^1hVnk67jdboM=2X`j@i|1LYxu%}qrXS%i^J&kzIjVlcX$Jq}UgOxej|h%FfG)01#gL+l@%yGJ zl_VJZox-@m)P|UyLa7gR8Cb&E>>Rzm09z}CCR)&+zAPjrEc82_jo!9f=x?5y{QTRO zvy-OS(TwQH1N+nVx~8HnI|kb?Q5xMx1P_p$NOrb8XpuP7$7~M<24M~ccKFQJ%INE` zV&Y^^SVi|iY7pX=#06rB1xAyF2>cDq;Zjn)6i)r&I2d*Ad8gG-M31LCVYPflX_wcn zkNS9WlG3P#0vqj_GFZbr8*;XPT+?sYZsDb*qOyqT7V>C_;Z37%X@rqD6N)otkeYEd zHqqxs`wg%^`M0UFB6xtDr@y>9$eh2(0l*}~&?Z}qVBo#9AW(?)g>n3%G}J#)*+s7B zU*qj-ENZ8TU&7*-OVGS?M?dNuAHD4LnJq9CpKxfrqdU+!)6sc+E7$igjbNCN^IhLa z@L3|L;E4Y_GEohyQ|dhPnj!aqCiWE-@B#I9T?o5(m4(zG72uc&>6rNJ&_IL&DJrVm z4V%4g!zFw#{@T2;JFSCt{)%_`$oA87&=mRcDaief*mZw=jWcg;)v7R zfR(>9lln34vbGP45=QOVQp)sim9RP#Y!RB>UOac&5tA;+keL!Igep>6VKvH zIL>C)*T(;nvQgIQf4`Uk{ji^fG&)Fh5z@ehzYeg#hBq@XlUD_@%#Umv6RK0t5e@pu zT#wUBaOxMgz8hrTy&4tax5~3rxy?k-^KvPE#|-xY1wsKMFy-czbP<~m#F3}{jb7-7 znO_7kTz85n6{UNAbWWmm9!H|LHfn={05)3GJXEwOz+!umTyThZfp?NQUU492^17S? z6#r2_iaDV&8Ac^%w~X5)Bp3b={Nl@7-7^EIvIJTgih~)ebk~Az5slA;gIerng(3v; ze5cd}{dNJx+*H>C_HOu-_v1TF7v)qY6C(UPx9&hl<2$x@PKx-jJZp${QsjB%`)UN% z(5$Ntu2gkEBRgNT-PMuAz?2a&hp+$XrRT@6n2oD5g{FBeR3PJ9cG=j+Wju-wj!&>? z?RPRJ^Bg6Ro;eCLT?#jseV_T3IbHPmI}v~8Q4;D;c6NNoM;HMb0ZTl07%Ko+B|j0x zm=P{Y6F3}S`y43~wg{xwgJ*9Nmn0p!r-w&%h+^pY!%xpnr3DBnfYdN+STT9oOMp^K z7l?+ZjMfB+LX3;WaKDJT7rl}H#Y)go4FZd}pFq@O1(_+>M9dfd80afOGqLK|w!DY= z)tb26L`NYGA?WM%mAz9dx zw#GJ$B!;-n(~$q5YB%@*aKOTbyIl>8uBiZ42epxydYo3f2(VWW2CsZVs4QQPK*uk! z?-ZwQez!0CaTRM) zjGo*)wkAI1J3Vbfl6kcSiru$@x>r^Ay7jPi&f|jh1gSaG>fOE4b;H2P8({4eGPkh5 zp!>w7WYYH}%kWSL+W3)geaUYW_*3ZE833hnMDd*gGD3Lk7@8}KM{bnc zMfJ%wt)aFdW5o{XM!_#>XL&d*vInuO7CBmn_JZNb=+~peT^HR_k=;2-z zEP@qzRc>r3qmLoj;q3#sNF;3i$i(zV{8^YUx!b2GHRoAcLUgn4B+D^?!sP0e_MTWI z;!M#$bm$Hu<}URG@_VwFQ0OICHBnT+HXYd}EEkkZfctS(6iF!o|q2^shUWr>cL z#yg%^82{h!-@;M$SjW>3PQ3FUQ@xKh?EjVA_Wv*Njs35y_g@wZ<1X}5F=9LR@lyLZ zc81_ssbd@q<1U+^;DZHF_72xn@8iF0!ha(FK~MhULz@t(54z4H9C=RVB-VO+Sw?F^ z37#I^OW47Q^}G-l!}T(4_>d9F27E(G!PEZowEimw@3?g4%%zs-w=VdkK>-_Va5#^ed zNJ&@aNC4G?T845mb;RE*vULW!&JO|-7ptfv|Fl{&z9C+W!QQX=V(P7yeo@ZUCd8Kc zPHB1MxLjBMq2{7lXRUNM?N)5iO*X!KZRFVfQ9SgP zLaD_6hoYyZa5KWw`;8Xd$)Nh04eDnyl+nVDU@{Oy@M|pG5KaW)klxT|lvrz^8F82{ zyW?1Jo;5!#2aFSvKXK_GawO35odiKZ(|%4|fNINnxrpx-8L^~J9D0SxtaQQ-wz z*Piq@>@v(8jz}0y<6^nX=(4Rpb6h!i-ImvH%iMm~Z5EZ$W3D0jsjOdbf2c$6ACYQY zd#+>3v*X1yrzBVGc9m&iYPY~a^SWZi{x3I>&9W)j|*gW zNVtyfH?AwRk>YwJm0o-$K2<*xrD>a#3pA6siHyVXOhWODf~Ol*#2ctC`r`vWmb)h( zG=KdP%9sB+J{KbRW)Bbidiejm{}6msxFTF@qAU+UDp!YpT=4V?-w!DZ)9f*tf*1xD zL*79cza~*TU!MZd++*78{^7mZ1M zm9?bT8Q)=+co~^z^JeJF(g8*t=ba?^w}W%){vu?P4ijW`EJ!Iml%2x$5Tlc6q_o45 zROX#T`l_4KV;fVn#L2a4)|4>hQ6gnm!YfMj2n{ATrPf*N9Mt63T%bLaxm$aLa}oFG zfVN_ljB!p`34A_`WQ`z#$m-CMtqpp=KslCB?O)?D4uEZg0Xv)|pSo6d)qZ~h7VBnh z#xB>k!eS$a_Tg7*Ti{|B%8@0#Pu1eiHV64K_eVT7?xA z51u)BgUU;svxao9*7S827}hc(6jPVp^~55dxW-1|a+qG}mREsxn8~mk6;xTGh(K_L z6DlqIk*N3cES!ay&R`U*YIiPTHAYt@_mhz@t4dHL1s7^$-#4p zt)k5t(8#ydyf$pZ5M%_B$-@h59#h#Q_Fi}&a@78Hy{m-Y@&!emQ)11YhwfZLaFZpX zmOG-rpe0kgk_)Sga5(t)V70jp;y4}$V!6e!4XGh60l0lG2X439q>Q~ao!wX`y>(db zRogl4>Hcc$POCv(U&$Gada@ z2FeKtLcA^6B$VoQ{On`$83n~7D!v` z>unx`9nG(b$&5U*enz)fAsloa>dXr< zk`zU}5820ZFG^Y1(_!YG-6~n!+4bJ%@CFTU6+3?sQ_;rK((s?x3uICbU|BUP2KGAL zVxa8PDG8^N{AhP03_9zfu*v1qaht9w-_!JE0<}?cq5Q3RG-QiIYa4pVD&ND^?`61% zsox!9xC*b%`2&bwGQFJd$!vPphEq(!ia(8I7v>X|HR-o|&h-3M*OsHkas#oWTTUY& zSzy7hVK+RUO8+_Iq)o#4TZ=35?&~k6<5?7wBF1zNbtv~2J8PxOd%G%g_21KPOFf2SfRNdZ;Wo3P z&0?my6M}iDFx`FSSqM2f1wHIk5QWTJ_~7A1(|(YcaCJuRpIDqYtqjJc7P}wA?$iVF zcHD&e1v0Q4+WSG#5C#jR%T%7)yBM6wC!{5M*SJ+_=7y}g`w-$983c8FR7!Ybzgbu(IJ`@){7^d~0oO5jpK`kEIehnnRUuB=C|D8qxx`~h^vUME!98pUTP;;e z4eD2W5_=3x6K$=Q1G{kgpPU!9-K|76@4e(ED2-nM^CL}+OV;>~Dny|*bVW`vY5&Tw z!E`x4h@5sr)LSBFKH|6G+2kyIrL2V!k&`y}^HXL1<~AMGN-N-4Ky~&7X(}Iiz$mPV z7nRLrqv8NvHwV&E2%Rx6;5m&g$c4)xEocnw z0@~=puehG*ca*_lQV($^?c_>eE zb2N5RDisNpZ~#E}x?3q{YjSMC;@=NzXr`nnq?7N}11LH)IG`UuK5b5g+1$yXpoqY4 z?7)VK(!(OsNU9joZ~n!kS_h%EWJQ094xg75ZVA1W<4Ox|lP@`rZIZp8z|Ixidc83# ze+q9iX-;~NxkB~FMl1c2G#NAMa)J^YQ&n-z%kJ36OF1e14KM2E9iE-TQ*XW%RM)|P zWCITkA!4!tGBj|SZ`#T)5@JSay0|yNSG-IB#=4yVb;xVyk#h!iT%8O>OgDoFTY^Tw z@99fKO;QUz;Prl4M)(%LIVK*Lb&$0WoFd8uq(zy^rFy28a?aQHF%71r`!fNRcqD{` z-#>7Zyg~v@Dt7qUqq+ZLQbsa!z$_a5wIwU&1>YTUOhN>6Pq8!1yu&Fd%zqWHKA1fZ zF|zf2H@KpK(qmV)n@XP4S4#`b)Xc-ryRa>$)}83;3IYFl>ssxTHFVa}_ORGY{{wDU zkUxI2PW9myQK0LECI8yXHX;|R@@W(qn>L~tZV*w#-97hxI=QIfy^FPeKD#cm-RR8- z@e}XVRiqLz#Q5YTW+6Lww~*A%F(Ky>0X%c~QKFSH3A&<>X+0-(jEw*{W{I)p0*3z}9s06BCOoLPz4F<-FLh zc~_X4iLDIgEa^oj$q+%rUCqTCmMnnzWV|cvl5g&F&J6Jq;hvRU-+9Teg4w{H)TT33 zRZ9-Dyii+Qh20Aut+C_IslC|llP*@!Hl%=Kney?X5h$2nB&$&t@is}%+BVNijOA9! zPEqqZAN)O-g|P11)}kg`S`;-hlsJg98aEeOkB9yHCN?{Pn{wM4h*alT{jOgwq9e4# zE;CL|DqrkA!At<&I1S;23{UK%MMr$u1kuL@E_R>fbKqpAA?3==hX!W-sy(X zJ#VC^P(r`LZkJK7Jt(^Qmjng&S^GTqpbwvb`7cT0vy({8Hi4T zsuZQ`(!w=HH?HUi%Y9@!?g+#>fN~AqGiqGIN2Zd!?=y*55*40@mT>CIk_u15oQj^* zxE%L{v6miW1;_ez$|D@+gPZx`S})fQB~Y=nC`- z$)&&-l!ZO;wXzdY1|AV|j5VW+qg5`7vM1Z0x`=4cKz|k>bGc^CKF3D$q<%DxM|}!V zHgFQE8FdnkO)~ed1RyMD7REU0e1vy}IXRVqHKaj>Tl*y?CQvx2XY+toW#vbbR^)!l zu7#LRZk7r~0pc~Rqb^2nvFD3B86so_F@c^7iB!3&VF~>aYd>l8)OoFr&IhctPP20> zbAnqJPE{&A8HkqPzc;h6vJ`rkv6_&j-`FB+K_hk4nLMEzP>>z;&Ms5gvX#UyqQ!Fr z4Sn}Q5Z@q}h~l6CJFo-5t|Q462N+ zZM#DWyJ!OHus88Hl+UmSH^EP&WCY&jw6W!trryY|9tnJb!nS2Nree&;iyE)#c50?P zb$t(}3L_Xif>%Euy{slL+;;=`Aj*V)4_9%9O8+d(QI?ZcU>1LdG1N?a@Mk1dad7O5 zm;#kKuv(+T-76|WzW@zVwxOy-O3JX6@%h`AG7^w4^wUrXF_ z;wabKj)VYj33JBJ#A?d%BWcQctwl2Fk*{#>|M#W>e78uDGks1zXG?d1}w^7-qu_ECjHaobgM`U3&4kcck#Je5JNr)R)% z#;x~*U|``5sdL)*%ElVxQ_Fk2t14rs?fl*mJyW0p3Q)z$J;i}bSo{j(a6!1mH3rej zt}(%Hzrs8fov3D`@&lgB%q|H}-^{wX8_2`5OI^+Om;VIuY(orQ#uc={%xWjE@2Jur zma&$md5ffEboB67$^l8nN2HDXEMIM)s)5a8>UqU$V45))xFhS1F<%;Gq&>`^s|6dB zjc>hOFgkZd`&~GHC`@KvKbMY-PdiLHuXwD%KCoJjcEj586`MokAEdLpfe=^t&=Pe} z5Nc^-d{+lDea+uS^hyu1y$lgM)T%MLj|v`Y>cEn#SJw^n5d}+?luqf~{&CiX zyooxEK=(Cg={~>VTHhgdb$_>2<7zPa41gx7V0u3QA;_XamTyugMiKnmar2jH?<_dP zpw_Gt-&TK5in*^BCWb}+XQdy@FonVZT?)Dw2Xj!5c66~-8!sCGhuT&a(hM+$6T!Zw zCwu;B`AzWc^n>d+QrN?kvSqVwD7EfY-50YpP6{rkG>_WNaq$S6eX&+z*+`4=v;-$i z>6lWJ=+=w;fkGyhhX2fkQD~=d%(fM~x*PX|eCok#_G|m;r_wWP2H8BzS;aG6$9(hT z^+l9@QLZj(J$(Qa=krrF$NzlW*JkhWQiw)%>j&>R>N;^T+gb+ez9p7P8DAf^Zzz& z`1@5x22;Pk3h!&J-`x=gi__rQH9uLIg;3EH_A{Cu_BJ<` zI&3|#KkKQAG@LvUkF>D0!_^%VgLit{0oywQef;bif;%)5ItwP(Qxy&603I}$tMvx5 z7NfW)ae5NrO5XsL!irRRs2P}fFcg63E_Cs}4~dF5g;~yBUj93qjZNipgurvWJf&lu%YCRMhk)jM9%j5zaMXrdI)HJBzucxG88`rW6s8~qlC9qJq zWKLqZ0*jA%bGMW=7v?K+LnFJ0d8&=gbx_b~W!Q>Nt6s|$P}gs-kGJ}~at%(P;75J4 zXW8r{QDE+m-&fkro8QxJH`ZOBzvtua(e5hC_vKsl|BsP-i1IC18U{u+qP}< zU$$-Awr$(CZJVdu>)qRLef@g%peK7rL`GzebMVW$xHfHGuksA;*1P|{4@cVa*}U~& zH&k&XFt!M%#^R8l+F)c@3|`EqB$%&NM+DlQB&NCCzT_Rj)J#l`m{*Be^|w{}DPJ3s zq;o6t-3G#1GC4O`R?T++OI0U3pE_6VH}o8rBveABUIqqQxL4402@Mj-P{9gB1pb>E zq}FfAt}9$E+}`^kO%C;0T=X5-Zu=|u3Zx4{wqkcLAYWx1)SX08F8D;NhjQW1(zUzV z?6E7uUuTCNF2`j<1hD|iC|9nJF$B+N`iDp!h`DE|)i*t=K}HlBaI z8q2xsAO#DrfY%Ul?Rb7lkwpzUr)J8f(Cr^@v`&XO>%>1F2*#_;BXm`EvJ!tuc{h@PSe zNSf58J0sVX%pSTX$E_dZXQw0_BtkYmAii71^AR{ODs`Ad;T1HR@H*mFQ~%NOuG8 zRzSC8za8A>vgkXWCU);5bcM8486)FCx+YzDD(wDmP{F!Y^ia+aDYZ8zHOk5@wbIrp z8M6X77)C)$Xso=#&BVrEg;vD%z0@moSv$NbBIm||o_IOYtYnGMiJ42Dh@N)I1zrYv zaQ%7=bCpd)FYv0eCcly5ApZnW;mq2bse9VHp8o%FbcheBos(v!~Eb2s9Oz6IgKMm>Zs3oN2X7!U7^uyQ>pPf>Y3* z&OVHkcw{yoZNcRzke(PKk3yum&m33%)x+fWCdMf=cSx7oLQ&Gy?i;Bam1J-uxnIemrG<~H*4MY`&DY`m5abvHxW(tA)i)N9b^SUqza zLW$@epLtUr*uJCcJvK&~y$%^~ye@VW?gSQj$zcmuTe!F6epTTH93Y8%g2=jcVA;AJ zP-Sil+F;AoKB5kA1V2aFqiVg!gH%QFpn3Nzl?E&Wyq=Wq-rGvmLGpz~oRT&h4=88c zDUmSUgcRpGLHdTF7$VN#+!=Tk>ZdH*N{IXU7%wqPZo7Y9?`1+dw&GlD@Ow#Ga)Ah` zuvQq0bXUrF05@z0UPz6>+XA(HJB@ex{B2C?XvS(y`{Rv~R%6$Ah_t%Zj5E#lil!ON ziHKAU>*vOFTsDDNKI@wV-ux2{8oVW^`x}lRTBLeCm5b zIf$0V$rYaDz(j!9JbVLbT1r?TXH@#mXvIR!xV>7iDO+#yo;2+Prj4TH!BR1;>=@b25C?NLX5$yTw0_}hrn`|`NNKkJnJh(e zv`&&Z_+{t3^!N!Hd(A%J7Srztn2px@tfCw=3G1SCOf~vuc{L6YETmVSL1TSKYoud+ zBOyO4ZL!QkOMgSEy)h6e5fZtbT6>=Df?>o-H2Z>SiC$F67ur&vf<(i;mF}s5O7m7Z z-q@>iPo4z2JrzacBr@^vIV3g9$0Hb)tjNl!&5J~O#6eal&ARIDvhG{Ep#I#6{)9m| z^qQlrk;L#TC{`kLTwu*=Ynvaqjq& zSVb*pqRwJI-JzQ#5*#!!n_)Amg_4@(sthwW5#PuS++CQ|BGuy=R&b4d-SreoSDwpJ zfoJ#zg`OBbQJ7EF0P$8q12qonTXZ1J+yezb;klSZjzJkMpm*H5wonv*GSs6CkQ6nx7pAsN|;nk)L~h%HaQVlXU-%LzG_l)JUjq2J2kpgTN1di zEt4@gy65Gkq4TiJfMv;5x<|ta>csxBoN!?jIiF2gZf$bgsJQcM8d%d4X*a@!b6EZ2 zmb{8<{FGDZ)XFGI)+x8gmctANTQFRA;GE8?ZmnS=R?o=YK`h9-OG%Qn(*}`qBps{j z>|O0*Nf*W`I)CgwqRh2--jY1s7P&A#-z5qUF|+)!PS`PAD2nBe0sK!%HNYBmw$t87 zmT~Zd&p;HNMUa{pT>(I-*|`q0we}|ws=TrU`NiZ76Glh#vOtI6Ir*2Q^XBkkbmV)o zU8Aec?CI#SNG^M0Wx-W%m3K;7w%L`CVW?B-Z{)4aDibx|S&m#XBFcaf5@4Q_2P4V| zY{Z==q8xio&gQ5$RLJps!q?~HP_swMq-Qm^>Wx#B|2n-2YkJYT%tCJ#iH=#=Yd~#> z9xQ=JoBO>tI|n71atg*2E2%F5=2=ZLHJz65Bn`u}CC>+(8j+M1dCPh4U=l!QH_s5O z*Pc0vk|E#=FH_NKBVn69XqQLGMW*b0uX!U6@w0Incty3*=vDuJPy!rBiS2S25~YB&K!=qS0;~x})Ks@jesfz+$%etCO_{JcYK{b9=_;BMu<3DTREl@Idj)% zM^>H%np^0ZWihej<1q5J7~V8ywdC|i^-$^FhJZ66dRYMryiJK+Z@QU~|EVV$ zXbLI;B@~OwV;w~X;#7caxh4ZzX3R5F%o8BVAi}uA8SoNrnKo}#V|Fl@RoV-gF%=Ur zuwV_e8ELDm37$H77z)2@v7mvYxBtQ0=q4Yi<+u?I8d4;Vz8xQ3p7p#03iX6Udoi%Gk0}f}k!s|5baH~{COl1F z+sm!4ZRV?y<+_#OyTDbF?iX4W7;RkmkAz;;WiS?}UE)Ehc4IFSyOiVPa{_sVE)@(# zB4!{5nc~zuM>Oe;?!QD*R=>bOR&ZGSK=PHKIBWkb4mvuqx*T!3k&1l8{Kz* z;ZyK14zdMeyfMI(Ct@K5FNMNla9s9Cybq*uUr9a5rRz{hQ$LC0l&7yN++WR28^E^_ zhVm38a&1rKSLyXJUe!xXvA}+g5d*U8X+F(w{984b&)1$Rlc_t0uo}lB<>muRD&@q! zf!7KIy|(d!Q%#?tlbwAfhBXs7TSuq6JPbnP2C14Uo_FX~Buh8@n&q`bxiE=MnNo;J z<6m9M#5_zIInkJdmLOF~@-$_Uh{3qF&1Z8tmEBGt*-J`Y11&R&==c_((SPy;?rfa6 zbuZir-{>1&jHo3y$rjt_waKbtjA`>%{`e#|ac3!%;9fG~GoNt%aIt|{5lb2PKayZ@ z%cc?=WI|DDP34i2CEB{kVzk~YbH=TZVvnj>aqHi#t1$x+#p|x9Ex$*WlXlP4b_cqh zd-F}*HMS{K_5TwO9IErn$_iE=)y3|05d1iMdpkcLxzYFbbaej9!0-D^K&#?@TIwny zm>8K$5QvTI+oUUAVZB|6+>TZ_RdHguo3a|i`Fp70*aRW)LR3AT6tv75^wu9Cb3D<@ z;tEoeu!>mm_IBL!V8N1i< z&>tK?Oote=D4o!OS#+|mmlsjd`aPf045j@EJC)-rKykUYW#{54{lPnH11KO#SM3C# zxf-ZVvO5B zG;DaX@C@=06dSqbIa;yjl!`X?YOConOyqbQ#KzN#gp>!9R};YQ(FGj`WzxCfTd@~N z*BetNPtJ_U@QJYpB@TunhFsF<7?eUc|K%uxM6 zxsr=!ASrSKlMX!{%(J`@+vvW=FF5`lf1wiii-O}Ndon;Wex6O5 zMHQu*#KYlHn(K< zV13f2^dN&4<|xW1!0YAa5fmXCJBI^=LAN?Z+REg5|9|QL1ze{5OTL<{_mh*vY4GABbeojCtYAAQvMWnsu z*LZ5Mv#akRzx+16(ak1ot>)hM1upJA#jHXEYQ&@;t|wc-?t+ldweG ziirTj{*kh*d-MB+J_90MmqpHBZXAR1sIuj;eBAf3&+Tg(QG+GJFa@%trA}+arJRqR z!~AJX{ky)jvny4?2!>_nV(6?$t37*c&ZaP)oSic_;Wq(|h1RoUQLYwZ1WI@iJBjg0!sqQNVZq~|$f>*aQkj{cigFOYB z(tWVm7e#7fW&B{MOaDot!id>EG?avOSCO(_Duh9nB-B@@7<@nD@C^e~ZeC)$3BfN` z?rCo%t=FYZcT#+Cj^&N|kr&@&6UXW7f4RYqy$r3cq$*40+f?K8~Dz>eCb8S#-x4c_l5H|a&R_ibgq5n5YUZU+9>1A^bbZ-Ucx zS@8&HN7s+r=cN%hdLquhW~v0Z+F%DWirWP?-@$XW8Yk>*;Irf0H!{>sHE-Q{ z#u<~aY~#Jhaw&|fFuIqyuf>`4t-7TZL5~ijd60N3Sg_TKke)*5)>P7=?VhgdBih*W)p0>*4}_BByE0NAIZepe^Yr&}eN<$f zJ1)xfR!NGpd{mU>tDF#Td6D_gkGXR~ob|o@3-F)%+;}C+e-js;|LI-v{h29_m=J%# zc~u0{cZkKORK`kcaDfMR&DF?@*J)+)joEVVL9Os#qukL;#aziV%hsFeNXpdH|gACnhGNe zF>mStBqAuqFmANXJRD%bWnHz&J}??B##iv|!V*}};yPT7WH1)4H@YO~*GiYJGU4zT z*F3$X!sM!VY}7(m9_=juyq0=%F%%g}VqT|*Lds+Hvn%B`ESs^_UcjjcRV4;9me-v; z&mk`C&0XQIuHVjus!c92`~LE9R1rCtsiHsdJYON4(evU&g5`5#K3GVMYL z#~?jzy3&#ew9v9ppR*bfb01K)Pjm%S=rceghU=0tNg*`{VX_iIv}U^2Kzc2fQnw=& zKx1IpSk@0j|KO!hPO(bY@@xFABs@k!)V1rWM|luGQJ5ti{A!yxK9H74H7!cOr2=M< zg+roNc;w@TFd$PCUSTvn{%>c{`eWeU;sMhz0b5t}Z15Rz9LV@+rQjN46pzF=G4>!n zntOuT@!MTAT4Tx80)CIMfw`3Q(Cq-yl=N}A?m@+L;6^W zSunQU{>tGRYAI&K&^>9B)~46{bs6cyWd_V(U%`@J4O9Bs&r#A!U&>BUvUeKoztsv#8PptaIv^u~*G z)cApkVn>Rd@DqMGO+x|gUe2nM&a8j(+7Y@nXBps|F;%g19?&3E8_tg80)s{lNL5X> zQTVkct22}X?`HBQyyPiVqfA`u5_6`KieMim6xBZ3s?r87j5HEL*_<>{@-{6UwttS_ z8IRhy=LSwrYn1s%ycN$nW4Y}vRV)0#!^Tg6Ayf2@Tz4-Mi3Xc)WPDv~5K)k25*=9y z`L?vJj-#1UoAy0Q})A0iP;L9rPS(EGpPEXp4deB}p30 zq#Af7J04!UnnGUDjW)LWF@#oNGQjcz97B_&7!1`pE)Lr`h5{Ezy36=}oZ6s@lSx}7pm6!4o zhPT23u`R!V*(Epu6FDFI_X}8mT?3>7lW^@$LZeZ?w4Z7_p9i547nHx(^Y;TdkSPk; zF_)42k8c<{VSQU#Ic*p_MVV0$!GYpoq=}9Ncv@o%E?3(2z!K+|exFxNsfC5eG#5#e zrnuWnKH?-s20K!qMfmc0$){SWEBHPWa2D0>wp5x*6Pmol>?9fq*vdrav9mC;W~wS9 zVP!^wqU_l*wPxaWj+0Y!bi2dC4ZIC}`){63MPvoVT<*`sI|2CdzBm=IGDJ$i=XMl& znXR6K&x|0p0eWPV>gnBt@064^QF^I1NhB{z_HG63>$*f;BOWJA$I!K0^P5YJ)4GP;O#hJoh7pg3%8LP0Mf?`0%Bd#`5a6*6+U39AGmc8m^h6=02yOFe9bRSs;?=mD8^^6IR!@!Ie(qYOH4zdbK_rR zF)}eO!>aWRNbUN!U2XkdQ;Q>D>PK@=X*EbJVR>f(|5=P^iBs>c)YxJub(;Q9ByU?E zoMWv`eErchlw_dlBYLa7bVPuER18c76pixn1eH{UDKSWi-1W)_v3Z#l_dgmNOt%2- zIlASxx>V7jKyV)BAlV z<8UD?;@5P5uB`KM``;q>9BLZh{nzVo78PC3)qlsR=zRV&_n)xmC?`O>#w7B0c-C|@ zNmiwJ8bUi!dP83F&)%x15X0k(o?r(-x9!J3N3B5m&2)S*cVkvVW5d-zmxEwnm~XZp zFEs5H?VPI43v*)&d^07}>~gjaN4(+o{Uw-W0qp@xP9(NUH4`Z8`B;ZN3UOCBOQ^LweMLGrRbMwakudKz$s_IG}HV z=}eL!t6!M_L-tMSRJ24hN^@VP0CPZf`j5AJw>I%eVqG=vCn)vg%w@xLx1}`#l&dt; zWSEb&P1~=uI{~r?_831X#)>YtgWR2A1d#mHwohoVJcIXy#1J8Pbh<+z-Io?5aQXSNKTF4I92HK?no;ns?6KlCyLgWkyr0^pxhWk1#;}+em z0rvpNH}kU>2<4cBIhk}gIxrHMb4O#f6P;2FGea#q2%VM(zBvxRvs&Zgh%vj6tYzM~ z`z41z2WNO)vpc3|Osw9xm0DtGl2vOsEvd{)BUTE6<+iqBp28kxY{DuOZA+zttx@Jl zGTB#*gEt(|5!}aOxCB{vD^m>#YoG>wPP=)V;tUnzVgb40x9^bD+n6}pkT5!eEoLR| zfN-|;xVA3>FO z)}O1hB7*fR0+I+^g5Q@uIM=H`tBtR+-{5LfFi#8sjxx7=ss$Z2A!7y}o%4XC13E8A z)pyleD`c;F>+6|jc|VfRC{0{ZsdNJ|$CjLIz9>D4LyKoI$)vpb=}AG^Vr-1^-?V zw($TYb-`BF#OJWo=A^}8en#iL3h^fjO^3ugoIG{r`$QxilHILz6X%Zqy(QSPUZPo> z!@&3T7zzMc95}UwO{_X-={nVRS)g}Ay%7}V8r_gs8X)pka@+c`T^H^utH!OiMvbD^ ziu$8hXj?3+5v=LMFlkk0*}N9ZhQh#6aXR(-Z$SgTBK~x&gayS4+i;3Z`Z>z*p(P5P ztB(~!gRQ56mlzB+9w8a8OjAs%wUX(Qn!YtmR+sTn57VLj=|pc+U^qoUUVi`S<_0J? zan%r7z6Qt5MzF1`m(4pdnLaI)tGEwJ^AuJ4R=qR}e}je;cKtp{<`2kGKjb?UErQYT z*OE+~X-i{hOU;Thwl{>?{-j6cX5I6mYUn{^a6yhF6Ayn|@|CVMNoSt}oHg!T{LO+r zfPPu3p1WtIij#AQ;VkJD=b)no;)+u#+se+(7koBg=P;f;3bcH^Js#FRFrHOak5r=7 z;?=MYiHY)sGF63aEh|mQ+;#10*v{!Y>Q$V!_YM+S(|~R>9>K8;c|TB4ESpo1P=d15 zLDIh}(6W2Qo!;gwP}uH>pW0WLnY>y20vDH;uOTdzc2Px_ot-NSjmd`Yp2F60U;bK- ze_RE8ZOvoC&V5wvx<$$y99g8h>byA&xLS&e*_4izgar=fauyH9wFK=1xIh|OT%%xa zZQKrwb_SZ`_l8h+-xr38IcpJWYv*;U^aJ``vVKdxluR@gOE)BFrbg4MYg+S2={74l zo_`^HKNvX44)^kBz)vt2Z)mqPa``zdCUGwAV7!lq_t*JBA0tmrzlYaLd{=Ttr(#BH zQ^NA|HeSXgcaxDJ%@W>~I(z}6qFXhn7WSCbOmH12QBVj5nhXU5dRMNx>z7DJ#^QsN z)-(XmbpRok%F#Z#t`!=hQ-@09)3mEWRT6AC2wwYU^UC6iVHRYH@tOwU>CPGIZQo$& z%ta^v$s(({nVAYb()SE@orv_*F^{W)F24&IJPmX#0246|(38%r zz^6ko&ZuA8@#&Dkonv=9)7FgmSHArLlgLEWl7cI1<-e#h8Jv&ym%)uLr*I?-ZB&>u zbuJ}q-f7KpKk#^BQ$cNqrmsOdBUM^#DWAS3-!q0wLvDpt`T;?0plZiI|G0}zFElpq z>0iKl(8J-rIE&{^e-0`MF0XYP==T?k4}Ied1l5Z=Yp&Sy>^nZ$A}u>jxFo9iTz{)E z{L1|^s{+0Ycwk25&(Uj?tigGEN(zUnep(}TcYrHsb={m^hK~`cfML1MDAO4?`-5-BX)CoaEabh zu`&d4hIWFyuCxn$O*L1vc<+Rjgt+L$(k46GmF}6}s3WM- z93cx&>sBz+kZo&73dGVqeCFq)TBU(qN5w5Yn`su-@d2BUEFz0YLo-@LHiwmj3G=MF zMP1HwSaY8qb!b|{Up`Hxm}&2#7%5m2hSY1>^1J_}OJW`l9TmmmsMcthk0vB`nheEFl3g^+bEwg4{JX*6eZb;;rw|-&WXSa zxNl2`bO}Ue23rbY#)QsbnsbSC=u1mKHr<6$a`p&0WpmP-Y8KU%hJm*X00|_ek^w^5 zccd@Z%ZpDmO>Z!lKW4pj4y(_wDH80oW$%)73V47k{xc3OZ4#i95X5eK3xc?K`Um@N zSN|j6PQ{wR5;h3n%Sz|xPg2PapGjHQAz_o7$4@_yueRegIyfX{SGo&J}bbE(>e|w(J5z3kZuQNExdG>p*XO02R7m9^d5EiA8fFe zY?}0@sM`ry6JlIIl!ua?g)SNzu4d(U*;{n zFOiv8^5&RJ`y`r=W~QOJTvL=0W^VR%|5a}fhm9ZnX6%fU;XQYO#cT?6h;I5>D($M0 zUQ*G(A_wz``enq3Jh4>$s09Mk$Z6jr6b9s=}7zXK{ zwk$KZiouSSffG*v+A?kS&VE1l8La8ncR*BOehLFCyw3C8j2o#bCdnk>Z20YP`N%fJ z?4>OHpT@C7zGpbnGEFnHSRn&DjkE{v8QGHr?0#eto0#B}7*Yw8_fmjtg$_G^JIMOAN^Zr$#vR0y9#VydOdX zXyQogk7OFzJq9xasQv>;7)g-lw}Nr+)jD_Dg&e5k9U;xEF&x=`go}_?$s~wQ^9%o? zfI(l2pTWFS7X=)qLeN^VoX!};vA()?X>8vQM136B_%hLP&YDTajKl?5rh?)StJQ@J z9h?6ict3VdRw@g7S`4vkB!=|}(R@RivyU;ZXT={1N;pSZ$6l?XdPzz`I;69&Y0{fK zFphyn(zfR5nHcdTR(JghLJL))8HgG4&I~Hq7NkimGwPQ+H%J@F(!bnqS8y4X*D|WE zfy`X08y8+ym5t1*8yh}$m5oQd$|-p6rGL3~=TuC~`ofe`aw;ZgXPH$uGAgfOdH<8f z`jr0VR!+(JaV!1H{l9J$`wPk`IXi&=?&4AUmm6fxPQE*%^4j^o1G6fo|6Tm=sa(5Q z(N-_?g+knv+sw#1@0`pI-yGY*B)smigkh?&Wbgu|?L zOfylP4uzPY6-RiQ^0*>}EQTc%|EA^>Hx48lVyHcV71lfo*fFsH1q{+9l8Fv%26itTT&sG;btw5G(PXfR$+IqH8aM>=z^VZB6L4QG!Y*xg&c>4T~UMjbGWM(~)4zKSfsIc0cz$ z+>W(NQFR-h{z!Y}JwlJZnlT!5nbbIHAemQ(g;kPWov!C;>2=X|tiqyW+INDhrAOV+ zY!A6cV|ZMJvy05&5iS0yH1E_*&ZeD66m~}k<;fxRh8BGaQmPLA(QX-xG4{7u|8v=h z47C!23Uy)SEvbemVG4FlpZD_~XVSpfW<@Vb!eUurp(bxMIy_m8RFB;sXDcXf43Yml zb`Q$q<=bN(1uw7U`ZTQFrb({;6uT+^1`cz(UihH#08brWeL{QK<8obZA8J4p^;fv+ zLEQ{66s;P0b9&*NTULPOQry-a7gOS-o6Ls3qXh$pa0L*R%lnQr+Xy6IOM8sbsdzhd z(VT4Dd5gxb?X4YtK~(Wws*$O!URw?M>n2Niin}Fk5|7XZF>~u)6bn5l< zbOp(>@cDV3;y+*{Y9#gZgnlT$1jRzWS8*~tWtoajMPq5Sgo4>?X}IgDbJ<3ZMW>>t z|D76-`PNaSPNkJ+5H4I@hCe{&YTkxZQgR)*xb^AV9kAB{~0MvbN(n&JmR4I8om zJY+57ia;PI!#Zge1FnY+d%7+RCB*{-goW)^I%jIhA1EBfP=zO?rGx9s0)y{V-e)zs z%(rw(|57m82A=BqD~`p*@JKp@)v9Z;S5HgyP0oT1?8+aeZBTOCF;c@|WdM~lgc0V& zImfZCgzk)1O_f}{=7SooaL0vTpBG;Gav^^th=+0rM z-~Z+fFS?)Lao`W2HGjXHucbaBYj^E#j->960*{1|@5 zR-QwIc+r4n*i$77NjiIL6Ge-`VL1Ad@r-%UK?4k|r>mu(wxT;Ze*ZX*GvcEY^j-`o-=&Pw}M+&S7trC!y9;iMq`cmNSLGF$g zO_XcZKxHzsjvB9JqyQC4Vp28;(e}B{)XLS@Iq3}M9CgnW;d|K^=y^n*W-f zgin0;@Xq3SvL;s4IpBtl+)313dkrk6`H?`@zou|f0GxV{E7c|$*AuOcqU3H}Qz9#h zKqg}Ft{w-|&wd$qk1JU3VV97kxb1^X9iB@uQQgc4#^AkWhNaeuH+r4Skl*M3o@i;^ zAozu=zSAWeP9|wlr>Sui)^PBbGAq>ekR;S!aTmn1zaJ{g%kKW>Kd%#7;D~|9pi2d8 zItPL^t?X}NJ)h^-@3|_xl&8q3mzZaBxMc6YmDx#5L`K_&L$E1h6kR7;U^5hGVOYFF z{r*eF?O6ArpN_dFnjQbqRHS5l7%Ow>=i~Hzg-CQ#v?%3r?mQ5{)!zREdq1Mv!aeli z#2~QM-83uAQP1R%;%nzBB8lLhL&O*v{6!O z2{rK1Oq?C8;Xe3z4t<=abhEj6r{er;a*09QO5ogJE*_{Lnw;$a5n4|&e`ZVqzDnd7 zIA5*NMaWz9&o$m!sCB&Q{ppMi|8V2$JDf>P4O5v}ZD-53C(Ph%g{F*n^iSF*2_ItfsTuiKCjS&I#% z?Fj)RlFlm(vLT?TYhFD_Jd?4=#CX$&=|x^t)mmb$D1c(7=QCL3lCX_AdLXaOPq|c@ zbvujra7u0@PQt18aWIVlGQ(-f7r&mBxaRfLpJZCqZ5f~FipR;d>5l<9v%RkGw(>& z;O;}Npp!>60uis8c_p^#a{Ls2YHWf8ZT(eaS z_dBy&j?rbjNv+?X%CdEI2~vHHCP|r`7OZC}2`&w$EZ{nF57mcqTWn~o&OxDG%W7RP zSKF2vxl%)anX_vH0ju$623D7pB}$Li`~69qC*)h|q^5lg_!UvNeic}0OKL~#t&^xQ zUy11eBnhadiwP{0n%dMBPV`(z%i5r5>^?Yg7+u+C-3UQKIz(nqi*!BTu+^ibL5-T? z8mJ6e$TG4`HqUrGmC0e(5T|#4$JmVgl+uT}F zx52q}U8MOC!8;T(zwyg`UVO{hoOH$4X!TNk&Mj>qKO;)liD=O!Czo9|_T1u_=8dIr zwONKlJHV}tP|#D0c`5na+NzW0EOTYk0WdVfc%>b)Qx{t2c(?DW58ePx81cZ^O0qA{ zTN*&DY+ArIfaN;Fwknpi6i7+?Nj?YqnwbJT%`D~_j0&UK z7@)A9c#+tIBWB*Q-&uT88af-AWdMRZW1sD(R=u4sDOfxd|D@42#zfnGwsERDRj%W+taX9>wX0=*GoiwKV+tv)EjnZwGigSmo;dIqyImGnFLOq3s z_C;Yv|7S+6k>!_1DT3xY(?8&7J=~l78^sbm6)20#+RkSX7ECtv}Ur{x)8cNHLX*nhfq^B#^vhD z0Hsv!QX(LW%LZ&9%z#u0F}JZhJ`>7{-d+gdYqLs{<`3SOj&9e_#{wWzcqWDnQ1XGG zkyJqIz<+@ti*~mi_=fYx?Y?f`2UaY^p7p+I85ONGW2fGcnWC(aJ?@Wcg4XCggm?}t*2$RfMJcJsU&;u`a!4%dTqFhJfr%t45DTyXL(P=Wa673mH#K$ zm9G#(vp7N7O7p>7_TKV+a$9GBuD}-Q5AZA^fUox;0 zvh9ZS!o8*1-;fuZ_hC5Q+HlPfoJ`G!80bN!cA1ai*OOTr4G+h_{-+^*MjOhs&GSo|2_CqZVc-@MYn&OF_4MZ?+sv5DN9CXZ%L`|vy#@u ztrCVe`%!CdVALI0STnumP!^l$o|E&dP=wz599NhIVFK)=y(^Zf^j$7m{;hoIjqm6V zyLCX(qn4W5L4B8ve-9wKm|a{BegB>mqPx}^)UAv4>wY4ee1E**3zuT8%C&1q%l2>L ztm=hrReQ7qS=*C^8`?sT`IcywuH#sN+o=eZMRsuyT!*To+7Ik3TXQ3V-_ zYZ;Nf94)q$`Cp%PfD-$7v2FCt-hT>wD{wM9koOPY>ORMq!4Y^4)s@9n3x8Nj!-X-%#|v8=3lBUcS*zE0_95HWZn008Lwg> zug&xr*41+d_R=voFMP3teEa-KeDj|& zA3zz;0S(_-&jFGI>v??p+OuPx`1Ro`bxQ{aO4?;hL}Dg5SI(IBjL)AWVw`xJbfNHt zZ;o8ey0E(NK@D3t=6Af}<_!z|eu2X!8RqR2ki(qpej^H`dD>oH3Qq=i38}4V25CPpVDgBJ9sxWW za^y|t?(1`x3%NLwFA6y>IQ+gEa%RkN{dmWk0_gHLx7s&mO3e1sbi!V%_*>*WN9&-B zE?dn^qJyedro&2wF~Fo1e(iH3SF^1m;M{V3hdr7~+`3oTtmj?2{9cC|X=XWccl){fW9NQ3{Gj#OD|k+axg@dbK|N;21tf2% zN7UK%^>X5?z*Kn{b69`l&GP5!4t%55B)%yNrO;%P)aYxiP1gq4HP z@@NskE1%_tTzabvAHWlzw@W~{D=%QpI@LXGesZ22gBXR$3_}2b^0e|$R)1Os8QPj& z5S-1zMu>4@4u3GtX6IT>fqOhf8V*?O-Hb^9gxNvti(IW{d1ZdL;Rzg|O`eAB_B!?I zWjb@1a7~MM0JnMnOQ+ZGNeJc#FH>4>oE75&s>1zatp`mz zwYGRQowM6t+uXSJyptxE7aB;q)Zpnjp$jDA%APaI{rR!Ia!8mhbi8or09HV$zeuF{ z^g-@Zod;*TyGHVN5}kt@OI8ps?46C`p?I{%HogV|fa2I!*Bz_FtML|^W)$YQuYY7? z3lZE++#GbYA4#gxfQ1nWHNx6wX#dBD=U-L_A0K|ZR7!Grr2sA?DfhG=598vzQL~9@ zT0e(nY+%-;o^`?-NB5WI`6+%Y*pV?&#xJ6mQ&Lsx^nyjt7+@cKO!~+0&5gWjfTY(c zXCOEs?X;_d_9aHFO8DPNx0VSHCvNRTbX~8GlOTd2<-f2xqMvc|h#*$U&XSAMjD%3{ ze=Vg8_gx(AVfYSL#GD|d3;I&g*Qi70erw334ywJX)!7{k6$5DX&5&)wr;OU@Yg zs+iq{8X&>&NjzCvVd%rz%Z}mwE<T0tI1{WNZXmAVji(N7Wa^BQy z!$CS}wrj5FrGU7LJ+xp@kR_ey5Iw)aJX+~5&;Y=qb6zuleLYVOns3bA1~ltCb#z}r z<2qC3zQ{H;1Mfk@Qq8xKhiz4m%a{_!RB$GIRK=OhKoTU-90BEdIoOi71yW*2F^cRH z>zr|_+!H-2bL9@4^cvY8l49lgq>rKGNR^Zk>f#eUs!Ha;P@Zv96TYiDa{nM4UoZ7v zUH_J^R-@97|Nb0YT+cbbE?r#v&7AW8-1hmHHSScq@WS_=qyHKv8r`R8kwq^#{GRIV zHFnrtbyFg}zWPY8mfy^1TiO&gyh@LJHi}}XRA_6CRyBpG-zEtTjG)K>gIlP*6-@o` zI6ISrJe&CYu7AmbpYOKfPXOsssfD`Up!YfCUB~Qk1fujc%njR-`!2B-?(y_N!G!GV zbtJFRFa;k(-BFo*_7ZkdNyI=79xAM@x50ojJbmL+7geXL(Hb$)LW@aLiX@ptr!vpn zK4)hSB{V#+Xd4}VMd@Ac3pqMGw3guoJS5Xlms2S=Mk}tUmX7C*27KSh9>r@n@TTl5 zKTok}zY!xhEgeN|c@fm*>;+6!TJ7HPN-KN>eSbebet9gz`rN^HH|GgmFFhS-H3|b< zXUr!jceku$mI0_sR&{UE(^dZU^jk?>Jrd?Z%v5UUcxKM|N-orglSc4!6ML8iZOdds z%f~bP`2uMEk#CF7B#w>JmA-ywfA{_$emNKUgP5u2R08u>3` zL`uVORG{fJkp@Rfr+(HOA%dG@YFg>eOBa(dNU2hU1)+SWjV@{iK+>nDk9!yQx8>(f`>s#KpTM_c>GE4uylMg{-@)l$5eeVr(t-~1bG$ithv5R$Jar4>;E@Rol3I30; z&QChZrYv6@&;0maA)!g#YrzR+ZJ*Mmv?=R0wgT1Zle~O-y)uC4~$ zN7g5JCRIFy*HS~SB5ynRrD$|a>skY%OJnA2RH_s)G=dk7Ul}sUhYw{Jbw$Lwf4vAa zD#S#{1TpE?_|Z@1C(aM+7ovihXU;-0tN~$E(@*lN(XOG%HLkYRCdmMwZrVky}P^z0vQ_yWJkF&JJl1od-EGeS~p>jRk{dFu_u`FoH{`nP6wCK=@n*?;o zx1v{>bZ*Bdo4P1SMNcbseF3nBAJxUQy^wF=K3OS0;h-5hygK5bys;(#FoanIS*`Q* zpV`P+&`|?=aEYygqdbxr;e&yXjGQAY`5lIWriZau=O8??LTk_~Ytk}7qfFXqbPU|# z`?j`YbjjUDdFHggPp?9jp7y*8uzC$S%gY-QG%_3 zgvfS#n!CSnjWHfJI*O))LrJGbUn=l{qEb$LSk*{2mKu%V@Ii*|@+>dlp@ciIr16+H zUgpfTz5wop;985?wVcR}bzlNb5%$%J^STGaGl)m7ue24zzP}8alUwSwgf*=rGWiCP%WF#JIjc@%D zDvPQJN}u?xH#EM9ySabXV;)vz(I1r_4|Z!=qb;tn%J=ngsDKy@5WL=Tt38&)yx!jY zr6tUx?d&W6NQv__i^2zs(qHt5($A9^O8GDY6~8X~9MEkp-O-O64(vy(c zxl;n_25!$M20?hae*TP+5)bp1%0`G)nrVTmth)<1=Q+tR_u#zGcX)_So_! zPIOY7IKi;3aQ5nN(Ob+a?F*(VueT@3|-GB^o-<%<59kTK-oUy{m=30ev)Yz3lh#*UwUO8!TXxC zk7gek!CO*v-#b&O%R#G3MfUHjH=h}ft>X#x5aDB@FSsf^lySDtw08#tIXHVIyv$`)5B``!E8rhiIJM?UU4YXZXbK@#VSYSxkWH;^B(kVVk?^?PooN zKW`7V)RQW;Cei$LBZaUwUH`F^6m3=doU}S@`*X#AeeFBQ|AtN$*|kKW;*Wc( zRY4tU9yJ86XSY%@1k|LJ7?>ihd1@j!Lzp}`qSe4%{Wdc~<&xopa(uZ}XkaRY+7&3; z;Yna6^sbea(5Kwui|q|9CnqN? zEA;W)Hn8TeHB1noGb|CHH@pno6aF!`y#cWhN zJYN;}UcZSz{&~Vw(@2%i-kfo${SD&6=BW$4U{ep8Plmx5fxlikX(73;tH@+J>?x$4 z3|)eo+LDlZ>mrGan;d}%m`&Xs5eh6Uj`9d7m&J_80BA4G@nQ~_(alPSH8LpKbbNbi zN_ilsy_La@<33;bt(YER{*R;p^!Tw_N)uH>B4t-0H^UZvojYNvOhZt*(v;Kak#3uv zt`i^eGP?zq%mZdC2!1+d-Qg~=f(qDm7SFgg zd?WxK29gTa{d?^M9w-c*G6c$5?4hf%+l8P8KJU|r=Rq|VSs$O%@(PN`#X>6+%nI#E zKXyQD&G}0Ej__+fS|!JyD=6lb-CQk^{P`tkLtf}o-k?}k89grsbB5+>6AFN`E@vnABxJYU$JgcMVgG1g z<)O849RrGQgU|IM26)Q8?{Ys)8(IB`xt*TiEt||biXK>5%>+pv5m&9{L6#>ho$a<< zFJU|Tp6G~%G%Dr?m60$rGEwQD)Fo{AaEA=4q?B20-D@`QRt}!L%g@hK?!*jMeqiBH;*njwNaa?QB^PZ~!3k zx^5xveekWIC7Prl%~Ld)pz89OhCUl8C`VEETy?DXnhtIxs{7P2lt;eAMw`)6PmGuR z>Bf>1_oRz%8Z&mu0SM&I?ewl&ol6TI^oI!c@FE&ouSnyv)K@l+L?-M2!rnL-$-rgYQCej~o$+cQnz zXQyXJxW9*y2mLrdx4-wDhlhm)_`U9*&&ThVxbOEfe32CWqds50&w!p^#>DZ52mLRQ z$5j{S0`tb~0|-3?-Piqx7N17@d%XdEIt^^xsW64fC@HTt?~H8N6EH7RtJb}Rj!*BedjtK2cVf%yCw{M?yj0N?N6k{gRJ;pUFEaL@ofs9 zRu0FZ(k~ab+MF>cnd(;={~PG5@Fs3zU#A?~ZW!litoD;0HEY@!lEEh;;2fma*Xaon zgP08V9=Uooxc^^QM*pl$nY(a+{nnV2YI;hf^$5#xYg_GCzdAn&N^O2lGUk;haZvPG@YQW>;@Td_8Cfep zB0D|H29A$|VUfV>WZPtI$KX#H>eU6`r@C9IQ?1H1GF z!`GS>b5eo|8 z>yyC8Wl$zvX}%OE&aVh3juLdnuN0-ZcUF$l0@*85X@37d{{$p6Ax|zO^TAW6DOmvP z_jlGVL{k4He^USE+J>Y-&iWEb{ooV!ku0*=U~tC#gG@RkvYF8JJYN^KRmb7UsJ5;*if*Z2zd~c5tNcuNKunix=J%@ zfK!^Ylc>ZY2hPGpkrL@KDEosS`3ExLzG}=>Wk0@$LgjuAgfJz>aS_Duv+K9I!Lr2s zbC!l~zyFgvS|X!V?Ge5HpsQLx|JyqlgJTK<8DI==38~V!+fE=mbfrhC;Vhl);c7Qd z_lsSHJSj5M=7t54)wj5kUwA<;RKf4J2E5M#ZwheR-R`h&c;p6OLE9xdZuXqgLKeaej)R=GbA>v`7ibguzfaE4r2ty8`a~*ZF1gK1(bu3A$uybEaDVMYxKO@jV zrN72v7S2M?Q^G`A3q08Lf0kME6a>-e%0ac4uw&cy(JK(mkSaAlL1_}9Pv|u1>`5-D!`vcC{{`HweOgob;%7yHFdO38lOn14~)ya>urk z9frSiSVoejoU&KX?rsbge*7q}fbt!Ya`FFcoRH(&XX>0B{U5na7v+qwX6*delaf`& zc&%|;R=ZYv+RQAf4Jm6`rbjiRM4(KiW~!?#(sO$uz(TyTs^Yj3F;paZfwQw;aXP2w zG6p`bNgTj6Z5kAu>S+1mYfxgzm>gwQu+itReZJD^S=AIR#Vk*rHaWsUDk*SXIdark*(wUEw}2}E=$_adOzx6% zMkS|*Y*Ne9LY5woK<3Z4kn&8bs@p(w*#k1vd^A%PS5K!I31!FNN5;>{Z{5e0pWXrO z?Wle2rCM4ZL(Y5nm0N6HbGQqOCHZ35B^D6OcG zC{)HQlN+!V->+QxDycpIfDpTZA<>f#6#cj)V@*dn^MX%4EHnQtA7|Qy8u{p_d)nod zk{YiExHiEP#E~F#yAdsqc<6itnjZ~WPPgE!0KkM$sk0NIvl%-PO`MPd4#qY6N}wr1 z*0MOF^R1vx4{LDDA}uXK1Tj>q-cysPxN+$LU&UV$!JI?bls3fu=jXMmtvB}F`JSNJd97QIG>b1;UUgBRYO zABPNLe_?k{N?X&0%_@@aCkmeZXaT#BRpQjLXbB6mjmU4Qq3Cg=d^|cfcW=kSo^_I4 z41<}Wj!h>&$+o|TV&w-wD7yM_YLv?ED`0=fL6h*f@T*IAFsvILbMCW7+a{96m^o{) zxb0qw^mhACrTd-=w{q$84U$o~9@z@@ibtLycmb z6tTeR4Cp8=duCFbi?Bs$F*Y~)*4x$g^ece#Ngz$J-kr{r8RkEFsk!{^V1&70z0ZJE~XFr6foK>L>4)i(3Gab+iyPgfXi6G=`2 zV`|wcz3S6dijPIa+_g2T1X9gKnFi#cVSjN}z;|Aiof3U@gm6AsV1$Y{KG>PwyD}Uu zSYfoT-*>(BfIX_!qreT?^%nNs=gFkH0oZM>ul6&iQc9Ch%Syz1i4(^Ej6TUL=0yYgqJ^9j|o;##j>t3;iK+jg|vB?H0h7%;if~zDI{Xm3`ZU zq?(y?Izf&%l!GM9g5qML6c6}`o7{{Sz;XwBpHk^(!!(Qa0oX_%6Vx7d1CaenJ+5Y` z{3?@@utVzPJnuz)Q3HQ zWz5^qrr>4A<#2BNP6t_o_`!bDW($C}vbe9I`JgbFiqx28s7Kd&{TM8(c(HV^+%qPYg}(z6G7hSPCQ&?=KTo?f1> zqv&TT5Osg_&s=?Qoza|h%_ zcf5g`}L4-#nmK; z+z&j&d6SA|KK6mf@5C;n#J&7xJmm>DW_?_9SwOXE!eMkm70dc0T>w8TW$}U)k7cmO~4pomLhUOX@mj$Yl#L zYlJW{=(~l-QctZ>34TG-HN_BZg{T3g4!>Hbd-}=GFnpsam^2UsW`=41vqEiGL}pXihl8={vdy4X zMsuJa>^he$qB=O1qQtA=Nf5osrf+4z z*jn-BH4WVXs-cj#=o~Wc=0; zwKCAvEi-BTE1xnDbq12g*sHyiK@O6}TU!4nBsTKoe7A)UD?swmge`7RH@kvd^gJ6H zg*gCPSy9Rtz)7>ABUkt?G({O~)DkIH9O&dm;K&65CmKZ=I9gfIBSwBUoJRI-Z!F;C zM!?9$V*V)5$?(9Ti>oitXFX3t!)dA#Ht##W!?T>5kL&jqHnR;l1I zdY`8kv^h{On{|?tY0?SquFqJ0@w}5=2<;G!yU+^iM3s#5^DRA_oE-R|`t9ek($aM5 z{9R$!a6WccHms*8GC>hirr-{>2enSz70N~8fIe4vp#I47x=Q)o$fWI;~;#KeL zL zs}juSewjQ%@-t)v*5iG}0yg)FF#g`fM(y(5Nb}FH$DHL&teLV_stc+Mq||wom80AX7n%OAYE4fM0Te%R3X|^T{1JtwS|`=lZBU|VG!iR)l&U@7Vf!V!n>^0|*>cjKL&Ve06V|G`Q|hCgMQLrvNSjZ{0orno?0|y6 zC~j08EQAH!d=}XnMo;o1QgR9GZmla`ClNR}+cclK3QR=(Y>(B^4m`ZIH`*3HJ@w#5 zUtr>K0;!`7#*ucaFg$*%SpX@qqfzZ4e*-(u8SqI%`Fgkh>mN@P#Nf*w^t7s1(ejS< z-aX2Ecz-P+a%KlCb3K5ZLfgWT*j&R|*C<`UHR&%hafWrgnU#boXb4ZhpiQV1+?1Am zqBMf)ox9#i>XxM89%jC!yx2~McG?6$B0ahQ+z~O(EP(?dyS#8%No<+F1t)n$B^7u8 zh8MhXSbmG!`*D1)>d?0hF6RLA-!v;(49&bUL)vEhhyH%Ld$=jS-`i8QAU5|WN7&!{ z;d|W}f0y6)?fd)rd6@mKuaC#?r?~I;6MW%h-Ge?q{!hQ&ABN=d`+Hp%huN)V#Hr4C z293G9gPuV8lDn)wivYD^Tb|d)!5TirUp^>ZdZ}C&(($K!Bu^H<9~k$;mGV=W`IEU{ zdQzQ&7+n<4L2(`T)&A-Neh|IXXv4FZZ0^Fr_2Ve-slp%n-jCex;hAXvf@z-*tFhtPZW%ue5DHsx(Vr7PwV z!zC)(K#;6TnYqB*Pk%K>rMe%ZFc?7hhCWRpqS56B#Yt67-{A$6@w3{jRCG6N!Nc<;sTMjU;D?V1q zv)E&@3RHALpbC%=s3q(`3n8bvm=%5_w;qR?ihS8%_^K#bl69(yP#3T?LEo0I#U z_6meE1OW0B1sro2y*XJtb-O{!SZ>&>1+O{zORJu;%rc!40Tg%ul-F=*zsr?q;!8O! zaP?n6?l`B1`SUK#76Ej<(U*hGV<6%HQYi{8yK6{Xw%s*Cnrf&AWQ!_jV@3*Hl?Xia zQ8JB`#|(&Mqm-rEVD0n$qQR;qm+&|Xghwkuu&`15ku0aI;h49wKAfG~w9Hb8SqWxD z#v|y}Og9k*x_(v#pT@@vNR1>Lr|`WEK?`ebYjqSXUTb=rT})eyEHba!WaoD9@^a}< zRuk5KDu_B~0;s)V0amVo8rmrQDa*K2bHt{G8mj=B5{IpW*OuWoJd9&~ZCLx|#KMcn ziUHBoEKlI5p>q1X;Vs5e2cmbfnULSw_Dw-zR~m-Z?9`~hXiCyo*0kX*1K# zzUHRacs(1nrN&5woCG9Y3@F$QPN6Obscny;voRbzcdI`WAq(U7agEGSot^a1ZdnW? zeu5UNMasKSJQX45{QW*IExpcC*4I?JC~t~f;y3*DJWOU5FLc^UoI~aTbLFS+dcY~( zP=EpoP8Zuun(S>R{lXz&6y-{s+6($|RBE`9R|3tJ%gJ%Cg3HU6m^;^~@SAgWa# zUo!1!F1;H`VbN@pBhC@!Xj$sm0+KwePtl>0%7t`>X?1W!W5gIEblB?B$y>+#VO-Pz>}{l8AWt+fz3{Q$?4Eke3=!n{2(l z5Vmg{l9ktqkI<-QH=$Jq6jJ6JgFE5G5U*Y<;nQl5j9Dls$;+(33go`C1d<#>wp!>S zJA9yct~U+CbOv*WR=x->1q3x3PX{|)d}9|v|De;25UzD8p08^$bf{{}T+C{xVQXLr z@U0vesL%Ik$ozF*t?#>ErZ~gg+v1}{Ex-|oOT;N7Sm#8P#&Ra`A?=}+D^{0hkf;2s z{Koew+)8Z!Xq%$`YA|@6;tR@%j?Jr6>`~EJ5EM0vo%Vr&)gm1cmH-i}ive~>cRC&N z_Im_8E#4_spsvcq;D*iR({|}BLvg!4GBrI(^zj7Kn2kxg3J^?&9SE%5lww+*+rjaz zb>3mmN|C?^V;(E%b;nzXYXk23@H_8_h+)M_62}s&oly=!|DsqG5ADp0s$B@WKh62$ zPqaaMqzrVGEr1W?|Tqcfc_3NZip!$R4EW076mu0`XqXO_QcBsE9DPVqZQLBxWF zcb&pIlt+50QHZ&iPLU8YO8w0%`IXvr#he{YgCY;#J}{nfGwggsrm17wY0LSxM!=B) zAt>u;9HBzP|YmT*Z1oP&W1Lcsp&ty=&eeRwe}CG@kGJHcIIPI5&R?PoMd)3!HH8i5ju?bTZA5E1hv%0;VQ?jEJvXFGF4c5-cinBdCNDYF8u)j z^VQ~winxrGK1SKNi8d(m)?RcgdF!}VT*1w?u^RLts%>W{mrPXaI$imv=mHqskqdZqlBP<4_4Wi{Lw+#$av7yO$}>6tHhiYU5^;LhMzIDU z<;woG{b+0XY!l+#2v_qz59R)p&whqTn9HOAKiEj}F-WYBo%s5wD;Tl zJ3-P~46=rXCRRkh3Xt ze^s4-rFrKf_S~H5J@jnJXZ3XtjN5)64OX(z0pJ3k8s8m|uJ=&wSb$FEd@pSEfkHfh zFfy32ZfGJK{*rh@jB5b%I+9kGHQn30qVcV(?(xCCmioL~mbdVCJK9<=OcqUdxCwAK zs|E|UMG78e#HP%qxdzj8Q8VQ_uPQWM$rOr&KT`aXX*DR15ateHjG! z--PW1AcKUxZ5dG#>#6lv$ZW}5zpeV?L+k^E6Qrj?TL!ARha)1DY-mN6RR`yo{iqst z_+-~3hN&`h979}Zj_0hzI4nh&B1<|Fy!d`az!ee`BRUdECfXin69#9yX9AMsKn!Q! zm~Vp(k^7f7{R|UdM-xc7&AS7Ib8pS@-rjMDvCacq00XEkp}IU)E(U%QFG-TJY~|3nF8Qx?y-wpy^tKU1H?g zkNwN@;Ur7nMr~_2f-S_g<%eH*5v$wRa4ELfsSPPW5s;1~ z!RlAJ6p@c4!3}FrDWVjmrd&vuPCywG!nfm(D^cX3IiA-d(lE5pnP$v_;-vdFiBHZHZ}CFBL^;StMth@u7&1o>SXRh-r^my&Y? z(P+Dk6`tJAL2Nl;udyNZ9$zAhi+k2l{#0YRv$JCrK^nL$2Bzf(wX8bsddy)^xoAs9j=Y>#pO% z&iM0sk8s4rLm#h5{yk~v*ww-+BY`2-k2><~T}7l_T)COW5@A3DtqIu`wq#JC^#(In zjM+9-Qj6V-IhbT2Yn>E2l4kZyr@T4&l{;rMG0Bt)OT)Ia9Z{Tku*n#9ifO8Pp1W8> zp)57qz{KJ{qoG2PWB4Nt&2DGTm`16;dENj$_v02A@r08W^L#znuoJX&AL3o+;@nnj9o%# z#u;i7D}y-H)kbwh1PszV@gslCTO4{SOzx-IWrv)OWWIN2l!5T6cQ+>?N$9z(m}sYA znKNMZ{EGvbwe=;$k|}4cspkf;7AD)*yuU|yt4x!< zfANv=QwZv9xlp7brP64|DT-?vPrMAK6l@$XUGKY9oDZ#rl}L9;mT(xFf)yyO7Wl{F zE^Z8OMS;MOZ|(3Wq{bHRnGR|r;ukfg96_-;tFrDmR@3fA=n*mfB<0ZK0n`}Qp=VN& zTYMdADvzQ+W06taVHVLLo;x2nXW$P6)BD5|6QLHd4>D9xbK~mzP~A+6xE6H`G|1oG z+m557LJb3;+@u&9L)opV!Th2_!zkizk+p0gBU7xfwxB1xeU4hGB|s7i%}by)J24gS zwR{g5yceLAL2o(Rn5NrfRU)0w_nw(tOp zkQG`voN8opqw(DmB(1U9+Gfh;Z(r;kr1&ww- zbw;l#=CQl8B^gnyz#*TyQAg3)7BD5)sx`6X<19hS#BdFXB<9Fd5-53ZF{Xz8QUoQ0 z@xgxps?D`XnR*Kdz%xuTZX3GY&1(bfKqdT@tH5!-bBJo-PqU#^$_IBB59>aoOsFNH z^^*HHcwVN?v_;F<$tKgOogc?yNB;73-(Uu%Lr;=D|J3*&dYIO^^sLki zetjV$Hm$=xvD*=Y%LZG%7qJO9NVSQqbf$%xQ;A0)#Gqlxt{u#B9DwM0^_^DgSM$0} z)|o&JZu+VOAt9RFHDmrTpEyPXh5WmOw&3^|HZ=AtM!aDkkj}bBIRpL_AqnP4e{c&j z3ouF$f}t*e0bu6Py@BGsegK+Q=(5Cqok7_@z#5=@eZWX=e`7D8Z!G9nPz622EdzT2 zed2k%BIFW$a!JoW$i=t>99}X;Pm#$d2yS8*8rW6QX>eUD)fCScO>X93+2Kuyr_yFs0i0F99Ls4!<1uzsp$6Nv}L{wh4+_De*8Dls564<|nb zc-SCSUWejjh2l+VQ z7Z#vXOBN+Bo?&HgBYBi=W$bp!Kr4XF3gBkSz;lFZWpV>$kQqFQGI3DC@3dBG5g;aB zB#b3uK*ksl!^ji#1ppb?Zl&m4Oz__2P_@X!O<&Xf5_d`pG>3~`$*_ferlSB;v7kxs zRevUUup|!{g_1cm1xu7$U7GmC<}p8QFK>MV$s+$m91x{w17y&|;{klpc^F`l7h!zS z_&YH1%P`(Zf{%^H(DlV|MFWX903oQXps^r=5&iF@fw)m1{FNny2FbyE@dnN7g?cn& zgJ>T2LWoHc2*YMzSrpM1!+TNVf+(?mv=L#nm@F|Mv(5$w(IQ5=(Lt0EA(R$_utM-{ zX!&MnP83;Z{&|riayJ|jMHZX|iD^2dK%z$mj>9%newYO}?FAMug78kmkirVL0eDv- zC{bl^gy(sQ^#U&exR{_lOkh?NXl52DqjS7?YDQyhP-++b8S~}K=n$kv0zt%`@L4uOBLpD}Y#fT&WF8RwWk?tl!B%XpbHT&q-ja(iaOD_aKrC2*<-oUyCb8eieg1^7 zyWD*KH-K+QYL=n_-@$TU;s9SF9Ev#L$0k4jyoY^0jtam=V}Bn{LoXm(m!MA&0%i}2k2U?TrH4ENmqa}W7#C3+V5^LJo} zIS!57{YL9tpWb+$IPfv&yZkIgc14W9GrW6lLsn~)=UCuX$cBcn{@dux=7cv_oMkx> zdk==`+1*B-1AwtJ4>VHgY1~&M>Pu|~Vl0f`zm5{cMGI=P( z<+EH_%U}=rXd_BEH7sUimO*>8&@euiK6*y5Yggh-}RTbF!zHLk&qqheFfw$)T5@`ml@vKb+CSj0xE`f3+SNOnIj&V-Q z2k|YR{9_$Q8d6q&X^eemSoRvl1l!YcMaQ$_qH~@tMFCQ_dL6nOzN!UkL-g+IxP9>T zar49OBgFQ%|q$ITc}7RcF=(7 zZSszUvCH$p4EhW*7M5Sor^Y_6IbX-5>4(*s zAC%)|^Qcyqq?D`~!-zqXFyy;I-zM0K;guN2cYS?o>&Zx4*vEYyEF3Klm_ z{;)!fh@PZ_yC`u%@tWGU{+s}HuU?Cfx48}KiM%VZm%_;3@slE zVyjdoMN<|CADqsT`2+Q^P=mSX09;k>FU+ysKlAgK{1+?2=`5XUw9H=e>CVbExZnQK z*Zj=m7{LjqxyO^II4g0!RXDME(1y^={02>b@?dDy8;}&8OrDInk4}D0Hb8o*p&G^_ zhrI^UD=&w8Qdo7(yv#{ic^VBtPBAPAp_A4w-8!#6SQ)>+4azn1=k-#J(Xy9E;2(A`+<~eL7_bkA2LB~JXdNi zr6yx^cUz&Lb~R;FFU0^2xmlQzaGi@pJL_!qqk~x4tY7AXU_FK}KU!vkL@ugZgvFq- z&R{|!;8I#;k{P?+TPQ9;CLOFqqOoEdG6}d8mv_W9kNt#a+PRSvKCn}eczU;`v=u^81N5-eng4@^1)_@X_f zF_)ZMeHU-sZOhbDWk+gGSU|m?T-PZ+jfz4Md>OU4CQf&{J7Y5I7KQ3DRfb*yyhP*h z4~4snOpHbTlLd?@H&lM|t&o?HOJYTT#S?t;&k({!gfv>C5;|`|1~_DKK|r|N$>mYu zUc%Il$+E7Bp>mox8V7H!3}Y4jVmCxN_{ARHLG;ppR0v<>ofw=lep|SIaqgfbii_I_ zV~?2`*Wn#1K1*m8ml3oHou#2W1qNL$FHb&i4pZ<>OiAVnViFI14Uw>gG!Jn6ghbdm z7^f?QkwF>wHV(`!lk#T;kp(fp-zbe8w*CHR!Fa5&@6Z;DZs@SS0?=jNf8mDsMbMu2 z&p1(>=4Wfi|CE-LnC=|%z~f{wYVcl?oeT$_oqbTGwd-_8g%+Ew(m?N=ff%o3n4jR* zLqFrC;jP~-5{ZW54Sg|Z`qnjazj0S8cg2zr7*Pa~#N2g8=dDj|)Cq5BmnH(@&{a@^ zhe83bAW-2%>R6_&ab2#z6<;v9lGbGS)>x06QG|grJAAa}?$3P8C7Kd3Dr^YW4{nAA zQxrMpEzgyiF|Fn*ya}|f!V*HL{5OCTdD#WW{!>(@(vW-2N{)_DLBI^H4fD`a&g$Tx zcO(GRxT3KFLeDnrpHdV0F>Y7OS+MBnEFw!5E2Pd86kIQ-LFMERQigS^j<|*hxMn2vFsFTSDk=hYlpml=7|&|G#ahE#s|r{BLSY036bbvlYY^* z!0-qbB@>!;<>X{cXP= z^R-?02#0m6y$u`mfDm=c zyA#k&EUKe&!^z577i<}r$mmMv-Jz;+0fE^wW$dy;Si z$Sr1{J|sAb6hCu71FnGZ9uz`#f0_$6b{Yz>R$+dZCD#0Q1rF(Rb{?r@&BF)~+A2Dd z*lNb@^7Nv_ADpX8*C5(s4f!Rfk|_&|sLngvUbvaIo`xg|&FZ>V%yTz&t#^-sDLO<( z*YS|PSNS-Gj`(fnCyNg`p9)^({=gzX5QZ*WWn*~cb@JEQ-?m3cYv97+KNc8U;$Ak_ zNUSmjl+xp)JW`~Xm5j4??uBt@$9@e;n_F(8toBCr+pe{?qJEX*%1$78FK^6qnvsbg z4O5V#-+R5VOx_t=kg%S!1yhRLXWZ&Lg&B-2gLbx{UMX;o({La(JP~-E$?C;i!%Wug z6MpGG;*t|;4x{(a+fTuw;fcC#YP8TFe@$<9Gmb}UgABcXXUh4>ZBj~VJB>UY*H`B3 zAC#@+3&>kVV~f+*%G_U33nZP`)NK6-#mBibzHe;~Z~%ceNFZILaXEyoR%4b4VeL5C zU!+M@H?n#RnO3s3H2y%*@mWq8c+}t#6ab)r?qI9EPRsG*ir|g{5rx=Wgj`U9dLJM6 zx#O4a0BDOcvU9h)N7Zwl}30bVPpAvrr z19mh-R^r%U`-P8ElBx4DL*w~ay307hHvpzBLUc$QY95l&ch$iWKysL5bRoEdEty5+ z1~wSo7O4dl4IC-hlU}@8jIYy-tU`IET<6BipfM1;?wMNoMOFTrPpwmAS;19+{rWhyri5$DBr2Mpgd+UO=J0_(XSR-y?GtMji^S zTvu5+hzZ10E^JiyqC3EQUO3-sx-fMeoxL@LhEK)W^)Sn;F*TK$7oo73zs8|5kB%IO z-##cWP5ER?rXN_D!Y)~3YGX0c6+bXs8HpKa4ig?!8Lo~#ptnXTA;j&m?=%idScpt! z8U~z#rnxNq0XmSa){Cs-A`?sHY8$#Hvb}}y4vKUfif$zzP!qIeJ!*DDp+uI$aw5ML! z3X9(IzwJtS9S{E+A6(m(^;nVuSMvrgz%UPjl`LZ82v_p$Za0KcV7Y#y;oGq~N^Vpd z)U&(y?01GJp`u5t1nw~VTvMu#USwmzPlCsE>_Lf2*QV;&-}m(--kDhb5aNG^*i~a( zw#iSn)aqHS^xO(uYr(Ajj{81sG+Ij4s{`sLw*c9HY*QscXKwVe{d8ZrooTW=Kk-{b z3mF3ya6?7s)DfJ5&_aD;{fsi2p_z0;qG{q(@ z?DeK6^}Pu#Q+E*fh5hld|NTUM=X~eHTRX{q|6TsY_a$ZQnlv@f+v#PqLRt!eDhbH8 z!^Q$RQGgp%byAO%e``Vwdh0vVX3rS`qs z1V06p*0`*+KIA=RLxSO7l^<~DBv)5>OJ7})3V7jw*WP*f1akPwaBO>))+kli*1t|T zTYA6z*yW3f=^98ds_@!<%3p$`GKG}7`&5$V=+$p>qmjtMB1D7$LqeF#^LKI#Qd+s< zAzou5P2EZYbc6S$6!qn>T8zSv?N>fQ!BSS~p8bY<*=t=RG7sSssb0+~3~nV?wY!$L z#wYl|@=U7kpHRELmRO$%5AP~PK)2CJQF4Dam7|kwFui90qYi}4cD~Ed%$%2h!mkHN zk4aoNP-_JBctFg4o$PJ zK{-cOoIeNXZwn49H9Dho_>plEZ*VqWELdhuZkvDERhAk8u9s$FT#A|mkt4Rqc#P_7!)OX@_pX=gUPte2vNg%bxVsnkJ3ouqpK^#^ zHD~WP=D)@JtG5LV&@%4tcFi-x>@v^ze*h#v+rPUAwx5NEh6?bO6Igg6Q6f)*b;HQ+HXF=4)qA zRcK`xqW+1c_+ZcwiH4RPn;bT$spE~#8hF}ojc8sFyActn-w*N!u`inUUmGk{vk#v4 zo1!q9?33cT#ihSQM&#bs&s!tW9RvOgr-g>tIjlj2JCs5ZGu!3X$Y5N#00o{hQSyXl zzLZ**HXC4|w)4IgPV5rr0nrdv`R|CRmV>VZ1m*r;P`@CbHiZiC~ zs%=fq1+iw-+|z!qho<}3Ix3Y+iQ{8uy)vnkt@ubS4mI?`UdSg_{3RvJGLxBU)pAfj zOIhpR93VF>bN%Ph0ni6#9GEu$6)NBXKMR#zU-5e=Iv!)-Lu2TgWjQHWS_Rs3z$Ag& zU;5Sn$Tx6BACDRbd4#P$`Y(BdY4at}H!SnEBoAr zHh(2dnZ5^Eq$Y;k9AYon1iBSk&+f+ZTeyDSjeU?nZos9$OY*CYEZDF5k>0`SwVn5w zz6VHfC~j$H&4HbND}(H4woBSOp4vASzMu>CvzN`D!5k{rL62U%!5T_4f6<+sijE z$gTgXCPPcUstQx#7}NZ{81pW^=GAFnA9Gf4DBbJCXE4}o=+@SGG-SBS4fYo-W3)4@ z14zZ9U=&(hJTrx}l5r-=+ulk#jg5H;4?71B`E(?SKah zKHD3BcDrUs*;I4)G|GHf{mAn@_C~+9&jv7&=8}t^T+e2vr+E|ug^WiY5dku5Zd~$2 zUdVf%_Np{o7|R~{S1kko{>8uQUFb|y1taepCRXM81-aN~fon!n)|W;yCSq|=P;KXV zn$RnVMKnqr+$5^(q=wrD07GBBzkaFj&7W^?uWjLq9^@K`x8p*wq$*FY9@AjkD8IB$)&h>~R{sVSWPjFjs_R;x9+=zi}rs}H+J zRoIemn5<$5%UcsoWa~g0Mq?XSKTr-ClE%hB%idZy_}<`#KKr}fhtqAnC#MhIzf)@F@}3pfpo`uD@xR3B0K-s zWXIeEq&b6k8OP6_=0t!`=+`{dZnIhQn3iiB)$sBXq%7+{^ok=GdKDDV#?r6|q|>vB zl{8W1^rS@aQDPc03Y)2fE-diiOiQEaV+~g_p9IRGbxZ=~(CmJ8lzkE?CxLPjC?~Qr z36!5d&502xSyApOQYN)nMwUEdyRv0iszRyrQH4tD*togl`FCJ{j9k|%^l{jXQ(x2s zS~MU@#$e7J+a9*jSS#YQ*ylU8y&2$U1DvdDUBg20;N0W*+#rv8BL21qZ;Pb1!#wSC z=4WGhSp=tw+_yLmHJ&>G?-|C17|s;Om4-P|m>WfLqJ6j!I)B0?_#6BCtqC(h{}F7j z4Uv6B8Wvn{|7Olyq{Pmj0uW8f^nv)JBZK5nzP`54V6Fd@<(b^tA~LQ7P^CIA7X+N^ zi(G6lZ?=?V#YA0i)`Ndj(o$cy8&;Bkc5zPdS_foJs3Gr$TrIJ#ZC#4HQJjb>Rc)9p zgjX_Ek}6ka!Dv_Cll#u4(|gJ@;LEl!HZ)PR8NKyOom=2d=7B03PL+0~6a=sZJC(eZ z)o}gPqwl50oE1j&vPbBZBitf{$m|rjnHtMyX`%$ zg@d!YUAsk-6a(oo{kVIJZ?Ge|CWr>_RmbMWmDKlU?-f3wWTSu47XYePo6bp^&-6t1 z(|S?+RJ@1JY1u0(x*UWwoQyqaBW%&q%N{cQ_68bXLysRQfNF#ZxLyV%IeyXGk#&&P z;i3we(-rNHtZCO*YrRyLWW~VwjY0DQVq@T`P+ccmzDkpFjkDfXnGiA)KTEfS7P1ni zP)#pHS5y0KYohPG{_!S%$rbcou(ChKp9^v^zj(IyHiu-emO#lBDyOy{E+bYTvpBx@ zEDA=`Z$uWC^z_zL+|@75Y;1g4RO}3YdMJ3wBFd(pENP~gmVYYY7{Y0@4n8~*URFrR zQPpDaduc8pAYpwi9SR=z%cqU2+qk0s(vFVE2|jtONFil;y7%#!WxoEH#}#OXw(Ob@ z|!W8*`PQJT* z|JQeyZ(pBkiqZ>{+11v4Q2d#DpJ^XZKkBv)&dyt(0MI784o&GcRawz2+Y(irde zy{7}g^p*>C{#DbNE`evi2zgxokZglQDORenKs#n>DyPe;%&W2?dx9=xn&sNh)h+6b z+|vPZR}wN`AC_r+38rhVMy5GPq-hpR)@PTx2D}10dDzf; z1}Q8cG}z9~t~L<+m`fE-n6-N|*0P}xFU}1n{gOT?hRa>(WFaDjy^7x{*Ne^>S#``HhIPqUTR52ELl+?wk()= zAqbN%eevW;kRtefG;?X;IhEu=%gyXwb42J3>53kDr>mKF5F+H7$PM?pg25#T*~X$Q zW00a9(iO-Kj=EPYntDGpM3P|gydE70dbK1HOH7`bYM5AgfGhV<5aeQOzpg2;hlEgI z(fsK=W%uVQr59%~1;msA2To(1RFR{LqdB>;^3j>UZueXe;!+lFB9Ns0?1-Gm0%nv@ zK{7^7OHo5;wpar8G}ulvri2cODx*UpbPTeC9QF;{u&1a@;>1!G2{UGBd?dI#ij}3w zy=&S@J|Tf8(MDN>e)TGTpX?jhKbrroV&v+TCGcJg|CFn{;6_=dt9MFs;*-k_{ZR_? z`j0neTBkqJrUfa_kQsaC_B1GW2khAN$EsjjE{PI!+|zC0M*mJ;(UPK!0~9EfL+lu$ z7GD<7JO^I!gp>p1i3G_0>a9#i2o>H-mSq1~&+sMwPBt1Ec_#%62M@ewG=+AdXZX+W z3tqAyHHuenq9wPxie5iU4;x+CL8N|!;XuRqQ|gVo_cr7}QU-*rOFspi<-&!=fqa3)?Sa5CZ{S zJAu8$E^LOnx~6Oz$%i4HpqWy**Nl@0Q!^DSPv=%T#3l}~4F^-B(IStNcy?@{6eO3HvDxIN3UP8{LuB&w^2BLN4eml5etmLk6apB1hDlPhe~Bd zk_ezQ2u**Y&2tSj+#fqHL@CB|nxc?D5$9*qx!t(Vpt`53>^y{0Z#q=f1*W~LP zCt-VD#E=~UrmfJvdaOvsmn_gtII8!f6gcy@9kNTx8ARmBr?meYC;fu zn;4oMsQ)^5H65N)PZ7nfqu%{_!IkW2g`6YElI9QK~olRKVt(z3~|k64sD5hw*VV# zgu<(0?yjNXV(WO)4pE z6ARbu_1Uv$&#)2W?|%0?A`3!E$`Zb5KNtjz5PoxUF(?1`^6eW4f6PmTjhykT`hf=E zK)XFX{`bu{@9LIT%{M`7blceo91c(Ox4-`NoV?-%6jtY^>x_-av?0SnmV`okKM<3` zy6qb4yqD1a57PwrYF(Pf9~yfzo|IS)7D=ZZXpTc{)E~~liGi@-Gm`N;MwaSNg{<@y+VkkYbEVvTxslt zzLtB=xQ^7;n6D6*DzO6@`{#uXXh^SNqSUs0_aea|nZdMXv_+sIPRt!jBPdU?vNFd6 zVVHW(OeySX8-%%`#hum>4MmkLUAaBq6dOs-%Uv z^ZNIY3=m5VjJ~Q?iZUopzWyv_6fHC73!=5qkIA&j)M9y4zU!bzI4pIdpAdcb4 z!(q-i9+@xW*poZsh~?2Z;y5*qArBvp#vQY6K3u*bsYn@@Zqp)mq?BX>V!tIKbMn>@ zWxQN;p=4H-2s{QE-Zp7^E%mVsdWa#hc8>uSj{sZu?m?nFMDoq@VhP=2RW6#rS|eP; zgPL}XYk0`y@%frN9^KRtb<#0qO&WVYZArdED4H} zlb0^x&IS-&iINpdny^#f!oc!1+pr?gtl9=c+MKsF6{%?vo3H{13Hw;GLZ8LUn=6tQ z{GJtxocuAAx^6lO^?LrJ%oFgX_&{ZN+>&PDUAcPNxJq2&kc#RoEOU)-ke~^qa4W0A z_`GcOmqZp?=k7~{tp`;7SZNic@EQnXTdsq{boUF5u3awu(a6CynX)Ukw+_lS;Nq^k z2jx^5Ya+PHhq}w)@Qw{tERc>|)rUsfRcvaYBs9;npz}RyuZ2w6b+yR2THlN|`ggsI z!B$p^&8)c z+<%88G|N=G9)>0RVHsV61Qm8+F^y{^v^xT!8z@!Gu^llR??h2#BQg0@5GgExGeWb> zey6S%5T0$p%moOc-Q;Zf;+-w~M^RhN5NUudeKJ(~svedRS%$TLINq2%!ppKO_@XMq zC_AQ?nLmfYvutc-dqyDFXw@Ca*vxEj;$5={3$?qryM(b83P2vf9|Lt;V8Sw~pyMLY z<7SgSG%XCDOb~S=5+q=r`--)xd=L+`NH4E@^T(PZS^c~(l-i$({+%wMEk5Qj1RP$w z-m#z33)W>|Gi9*bo~$5Bbb7f$<~yPWxY!;}HMEokp-xV<$*J~bIMuGusirw;#uXIR zY0w*6hAM2O8|$}tRZ}_?Z`)~Ko+Lgb$w+f)>r-r%DbA(oocu^05P&tHz;vwnxK1I# zX5Q!=^gNKc95CzQ0AYs;A(1nxGwcwq&QOcj_zIF`P7@Za##Il;SR>5K@f$^;x|0=8 zW1enTV5+N%mZl5-hQdF;*8A2|VXJF!MYJ$iGAw4oij5+2dDe)_JG#FZ(B17A;g%v6 zVM4|4nq#*Vo->teWT^{RN2J2nqjtc;wZ@%&g5|~;s0oNy%ji>#F0@Y$Xz)SybR!PP zAHIwcb-io%;hD3V`Uh&8j zM*DO#aiL_JTo$z0lGK>Swj=6acm*&HF8T-aOL=if7UPDcyxNQ?qyf+SJMsWUYEAK5 z8&iRgg`K#js5wpUP)l;#iH%jfJlmrq;|Jd`I%Eq(Q#1rZ$du{*wt#R>4f8L$P$TZ@ zi2-t|<;1rjCOrA^k?0}-{73di%Q!)2)DV}9=Z(=Ev6b*=r)eWqN(p+VDJQcenM`OF zQ7(@(r5He-*Y>VjN1#R_{Y8}iz$}`nu)&+=dBzhch!2`79J524Td3*#k#DLEsgy=? zDax?yq?Iqj9RBqu>)#7m)h%t}|GSmOh8hfmVZ^4!ru+o2PG%V1sO*%}< zGFCzc3v*@rF?wbr?=^D%Ajr209j*QDndJ*YPaD~x3e{J^Sc>^E8FS!^N~klkIJF^y z{eb0?2(2Kv?hR-~Dwy72sAE&e4N-#TYAvDHT|#r3@G{U;uAs@CeieLnFugO9oSN`v z|Ja%-KIOWQ)v8~9qweh_qss1XCtRZ?E0S&n9ZucFxH`q9@&3R(2qo99^fLOJUS^bU z$cbW%ytXKff4K>Z%V>=(nz3TiwT}{53(O9b9}S#mcX@Ro_s}4g}#s>)~ldGh_rf`CgDOZ>|t5U#-j8 z0|U=4^7NY4*h%tsfNK9OQ^06SW9LI_yk6o1-9p_w6A#%4F&R*-4Muih04(zoE^I;t z2$NtG*GH?mo3G*6U86<>uTZ%A8CjyD8|yn#_pIig z_f_9%D$*IvG}>K%_i`vwz}?hFDAuhdtHO0X(t_HkJ4z4%*k-}fwC+8Frj!T43VSQA zzk7K`_?*q{{|bV%U@RLb#!;kBB&D%a)%k(B^$=vkk`GATsVWKPsDr{ZL)a#^-b1v9_+2{#4GH16< zXvxH2z_QJT6|yu4a%j7V%J|< zQ;h7&)4|2V*-&AwWIL7{)-FX-i$ZX)0|jl~fdrIoP1F!t3yfb!u!Jc^i|rZoG~=a> ziN=W9@Lcr?Y^cEQVFgR1P-Rgia8i^k3VFwbOHI(+;5H-#5*rnOCu1X6A#_8MWKG42 z)v4SYngWje#c2RT)(m?Zbg3n5Uc#{~v_RrKUOxy|)AVe(+p*n3p7k;wGQv->VI@sz zNk>Iyzoo8omWgJ4K#wAjjjYD zc5};z4bz%tuUM(-;7sIanlKva#aM2D*}*XkLZ!x7(XANZAu4J3lIFfciyz~n!+vmY zWg6$tYqf5xU#^ocFu6y8c?%ziayx7QO4;slOV;#$e@k;Mz0O>x8JN-xJ~z9Fafw7k z6xuP;(-xDjYKNv#H_>jPR^=b}t|9d%Zo@?eMYN}~gK()|D;!vD3(MtYW5--4{Zc`>TLKia2SOME*wEs9XXjCzQ zjw|T;I&Y3{5Czl9XH;P|NgI~{W-l9+XulcfIIQP|4Uz%IrP$(TzVeU;4L(s& zS9)1mp|X@>40pPO%!@5Sw@^dP-wOJk|=e`>;yf?D(527F|`4d8PZ$QlLMNcLX6 zNR!T1$HV$|s<;q_a%Wb4C7jNcL_~Cqjp=|$T8IaWd%8u9F0ZfPWvdsi;D1miniuITr$xC% z8_k)UeKcS{Z8Gwu(J>)It%Us1IB(cG>oGHKRu`K9rx(4>`jnq)l`%P*b~IM$x8AT8 zm(O&S@i7}vy@Lm;8~YCqrqKJ7jSTKMEP$FcY2_hulP0yQ83+}-0@7%lF8w4pC+mX! zL4WOx@O}%eKPVAW%uKSrbKV%a+eQ$hO@WFy$<7FEFk2t|<@J>z(kDU+BzyR5oZyC* zv`*y)!8qx|=99;ASg$OT6_2<`H9r8h2!;$*B?*hXO2%yHcS5dNfZ3i6Et54uYF2s>ki%4LZ+aecT+s?C~D4S}6U(2|N#sy989%zv&C~!bajvd?1P-^o9EOC4s zcSY-C=AtcU`inlP{}y*m+cN5@lYJF)J;3Tshyz-cazjg=fCZ2*J@4Aab*Zqi6meG+ zL=g~{%wd^3*)6R`;(dD~J?z4x@%A9@l-xdHDMTZWnF-%w2nl%73l@1fH(IHq&<1yD zYeG1esW;)BgN&0Benq|qAG!w#&8t}^9~3Dcq_cXV^cQzrOGAy+SxX%-aJ6`;k1`qR zBkmvqxHEm%A6S;@|Ar{`fayE1dcG2v-OrUZO`7DIleCa|9xEtmTt98E9>PG1d!`^5 z$R_=S+B5iL#RMy8364)w!ILiu-I;3RD(t-xJ{q8t*&gXgO6?uHfdJ-2jRE%ZMjjUw zhB_tRnilYmzqq{Y9{WI?5QH`av=5-?#+V4^ie=6ue5psVRwamMCmMYkaS$oX7l*m%+cnx^} zSeXa`{ddSX9!135&d$jVvpsYCMK^=MqG4-_owAbhOx3*fwwr%P10BH&$84*|4}bR7 zEtd#4-ZE9^mwLg^en7r^AlhNo<#xSVlJ_i;_pCq!c>OV_LUAd+raX)F2w7CAokGy6 zt{I;&(hKPbx;gHR*#J}fOT#pQp?SgXxvb*$!K5)%B}y2oG`jkluUuzW8>$SUH$@t+ z4P!_#Ei2WmwUR(U!y~%5-Pu2d#EHomvFvJm3Ax7G@tLt?&+J&TUxqCCT$3}J;}Mck z$qwy#gBk7k6uGq|hoz=-f!S2ic z80oAaE){>eA3xvw@!pnsXx$Lx%<+fo@(OovNTW!u{oWXvTyA(-G8^XXZU)4sq>=UY z{9$MUI8ZRInSFGYI0YMw=PB#7%QfL zo9^{TEgM6%tG)E=Hjrj zh=xdT`y33Q6fSJkS?kFi+r}FQi3z+mt%5CB%s6PYHyZ(RxTJ4$j73fyQSGbz(bvif z&jxzY#5D@?2bNejM*kXCFC6yF4%h~F?2RPgJGLFD=)60ys+AOK&2!tn z7w);i2J#)v_!unmSBT2)l{h2sWU2r28Zt;m;-wz1BvbEX3GaWBL&D`Ba74zx$HA|L zSzb_SW8^&Qxwu$Lclcb9D?wyoc6W~}U?z#vv+jl38$J?Z$NbEJ%c60Dk2Y2310P8B z*&PT6?57Yx;?hE(nIA3LDk9LTIX0|VL3Ow!s{K_xM)D{ed*b!D@OwMPm=|#jH^6E> zZILEkLLRH6j$b(lLT@0^>C;b^*Ls6`r8mX8f8W8 z(~SqGc9xmU*&QQv6vzV&)lhOHE(vQCL1?!WIbid9L3tUw(FrUN?zUzoTdD=rOJt6` zmgdZMn=t$Rx9Vi0D44P^3DY~_$ckFOqem`1@=E-^&6a}!2LFl%UOZNv6T zl2=78mFfG`zB6%@le>z>Ha!%xmfZz`6jGD690l>RS) zbG>eGs+d@Qe2j57#-<*NsalmP{TQ%PHWvcv@WY+0?NAJxaC`mI9WNPIOvWEjk82?- zVesBo`8s?@_3CX+0XIE!8|uWsxJfScKADOERx|MYmrGU@cy^0zH|2`@Oez_kG<@j{ z&#pU$b4+%eiO{9wn&GuoB`w(*_Cs7|wrxr@6WYj>FHHpDjpqzK!(tMq9)w?nISQ=Z z-p6yI;(u#`mqedLJDRaOj&gq-4Mb8t5hoq$uw+L^UoWU`ZwsI`N^M3IkgRWWpak1gFx8mV8kGN8?OaiK-A1ip*# zLSlu(*(9#c!l>0pO5QO__6$xZ@Ib`31|M1W7O`kHWKEjp$FKRKVC3bR3c)h35Hq^Q zdgn~bk|t~91(ayIm9g?zk+oDM+0bOog)5OMTsiH5{E1xliYhT?>yfQ#k4z*M4ur;X80&{|A!${lTESFquoX?UAk1sRxLJ%%*NH=rS zEORX%NGiz#gFqxLr$T0dis_i+tsau{61m9>x#2cS-r~f;1$eV6*b=FE!dWHs@z$MY zzJ5tcTC7;9;Sf=67A#oQ^l^KPjc(m}?0R=LR(EAB;)u4o?n$P!a*CC{oA?=6k>N&Df~t-UJ+MZUvF>2<)!FzA|c>VOPnvZQqu?ENRG~Bh!QbpLuvY^3_CvvmV zf~~lKio_v6o=kn zv29gL37x!MDNZ29=4Pp8p$jxn@+A@Bz{Te58ri(oKG(T`@YUa74Rh|wTDA*g< zHpo2NvS^qejgaM8D*CM%Jr)deXAL*CrR?knjB!0c-HLl|6FEsy3DJLOQKA_wsbb}j zIj$+6)$eHP;olMSC-6V9l+Xc^VQSBk+3$^*CmK+evTz$?+YN>3uMMw-Z|glSSTa!k zI12mCjK2N$>ebk7((hP75dPiVstAuCxOw>BDqjVI68~;<46MUeIe3d!39C(}ueiF4 zYs0nkTJviC>E&x;evQ;zy6uERhQfZZvaT3^U{9M>!fIDa%GKQ=t-ba)J+}1)+j*jH z4HLw-t*pp{DKKvIlv|#^Ag_h03RX7-YJFx1ha4fkgbI=|TUg9cm@R4r13KHMB75Y~&EI)CN_08K=RKC{o0qO%)y}E`wd7APU(d<6 zMwKya8L66Hzs6~J;!|#H)YUz}LJ*_Rhj~Ct*WdLy{1h7&wkHg0CWujc%T&C4&+^`I z!e;%giB>n+h0_KCOw2iF_Gqyi*SjCRv&Hqip3rKgV)K+)L#nAOl}`bnafPBx_dB>*oa?ZF!GGf z^bQ}c)|Bfp8cn&j8&a>#N%Q48I-B4pU-%;nh3(xK@^35{UM%PjSgyS;TsbQ?9CLpf z(qdRg>Xjs)Zi5dMBHb117c=|2@DGwfO_8nx$lW|!`*f2C<^`L%?lViCF{s{cd2PS% zR@9~*YniilJbXLbwnF(lS*#ZqU))BhG&M<}Xl;j1!=NVgt41sGd zR$dh?zc!btSJyP;zt#V%u5-}~81BcrZ*HM+FZhcOR&?}40w_VqY&HXv(#d~mnK{i) z$@k2BAJQAjf+r<;SWDL?71lO$S9u(SI z+nrSCVdOlfRg$Iil&I|{INkCpLpdFs{H{pm#akaID!k0HmI#2S2r(ju9|+4J>#b%0;q!u za-UzMkD0=19!FM9)-=DY%JnO*pa^5UfU@(ZmxFiN#;ge4KC<7xh;CnAw<@$9UyZFOkwUO!sA+E8k^C*5?D4;e zmZcPz`hO~#;r6|I3^QMI`{QTVc+F0? zCV9}m_vkq=StWdN1?PE-$MMGHz=Wps$WB1O>UfKmdET9mKmTj^8+pSOgyszOqlbsN zGHqc&a=X?_C(r*%)3kVjJa}m&sbVl;+_qw%!g;Yz%583X9o@dX1}KXB%Wt27B=^LixEJMcvIPOtuSLrK+}0?pxy`BU2F0t+TTkvmGS z9Cu^Lb2Q!&H7r;h99$Q*qr-wrF(w1^7@xN?i@rZdd zXJ#GylKH%=6j2<7>wMLe5VBR2YWf$rRwjeZ7BuRrjb!6!=*H`&)sNqgS8u3YW(vx$ z97H?;!no*A_ewkYxM#bD6q4PoBve_hYr@z3Eaj)5%m?o^c`~}~$oBVxS)##B>D2ok zN9P-+84hRmrVMLYO0h>~P1lP}r~NfN@p8CeTa6JIzwBH|&}Z22L@BIOsp(QW;Ew*B z(&``6q<`pD&C?NyZ2Dol=U1HoA7nA6NJsOP zRs@atTRVxgUmN+n?%oaDy;i@yzB?F}h@H|p@(PCiAkOdBx`LnA)!Uy-4pe1uY%{o; zy{a-CtbZWNkDxjB1!Kt22Uk-Z3PvmME`!yhzDAvfo9QBX%2{pLuE0Wi4OHU5L^d7UHv~6yfn3+ zO@(Mr*ABZ%MD3K+WJHq;^A*o7CWuT*n%UNttbAZBEKNRp_6%*0&wls2-w|jKma>Fz z+CT_C^_z=}Ir+brZ$lyd#I(y*Sg{Ic1P9^+R%hgY-+c3~u9Is18U$=A(Q6!W+c_t% zyzo2U)_u{D$nrnJa!s#oKOYV1-hScK?toskheZJQn3Bhy{W29hH! z7`O}(PQJ!BuN_o822(($COkBgVg;u*;2)8Gun|jL7VkD6@Yuxnj7l&%`?jc5`NvA6 zSw!L3#%hmV?m)oUUXz+(+gV4|AJ%8|VGfmX%NpK?x~1*z$0DE_)49G+R1jK~at3AS zZLsy6{2&O_RX&5d$1#P>jRdtjAMf=2TFNoCZhsIR@Z9Y6^)rs5h3%>pJ5@n9>`YTz zTj01^El}~5UT^@`Bd3c8Ej$DESTgd*cV)-?>*j9kBd z3o+mz5^GVcWCD{L<7$Fy1gyp2IU%o=ETN*GhHMT&y}=_w*o z0xyv@qp9sat;M1|XX%`fm-;d;c|yymW`_Fhl%4(6| zGeqjVU?tY=Il^@a2^kqj@Mt7xXk1X0SoNR}FuP~P*1(Q}JHEl;zWq1Hbl{8zNdoOF zJ9==AfIGTX(H=d{o}v?HWG~K95bu~FRH1I1V<7^-vHdvr?Z~-DPtGUm%K1orIS*)# zw_k70C+p7nMEyCRqC@8)Jvt9KuzmV;K3%8Ir|Q-D6x}+XuwUnsbnJYRo}CAD?cB3( z=K;9BU+>O+x_6H4-?>)@&%Jth?z0Db^zl5jljokN{AArcpRAwf6L<7{ik_bFT|LKu zH2psxp|9s2_hR%5hjsUiiyOz$sqX}-*ht74D;lbO+_8 zvdic*2Gxdquk{!TC$2{W&)$RPac|s5(2B{c+Eqvy-F91offSv83s~us+^PDoJ{N18 zXTF|4j`8f)6iUG=CreXk-!!D`n0O>~o9G2hG@*Gs0-H)*{#pgq%p6 z(gLTIEMMuxI$v3v?~K?r;vuUSFk$>IF2+ikAfs4_AY%$I0@&K%%w4lT!}??8xZMub z8CQC`cWkS6R}srk>{{#8aeX~Zs7EEdZ;aVb=&J=O{EC_4%GBERyd?ECPhu$+$Hu${ zqQwMv*k`dt9qOT2S>-rgC^4W{%ap=~%w$80JC-`hD)yAF_2xPAL}8g8MB&kGqrhgo zA@_JbKSsXUQIs!0WfO^4MS4$u!okeg*!SrM5oZzO=8Sh1Djq{7u5nTgOTcf~+ zBD`8rT%+IyBEDjpf!h%q0=BcPsF1Msh}Xk#G^0bkx}!xyGaKa*_KpO!(cB}RbMn?O zyphO5wG6CeR+X*<>fW3a2Q17Q5V4ZA1zB^`(aSb0F`fyVSI0Amu@?6+*7its2*;k5 z#8~V4o@T(oA`+3S`^2RtaEvWD5eYV?ECr5FXKJ*s+O{XI%v?i~k6fTEh^En?{8(`t z%h$y*|8ZuoQHXztHQWZ*UtV90v?h3O_Sh4=C%#yV!WCK+yZ|Qhs$SN2W5l0mjedXS zX=iF*YgSpuaYVBX|3FJ=QWk3FTQ>)dcM6&OVIVY5sE`5tfnrDP(WLJ0E%!54RkX&$ z&l-ocEM>!IS3nTXPvI{T!Y?SP{Z>&|#9}WsyF)-{dsbtMI zbhMnM$JhE~u8nwEM>3O=6&pq5a(wWIpRsmye-RmNS&zK~B$jq52W{=OS3nwVmY`y1 zW0fD`RReGqG)?U9@$5d)w(J-E;kI(Lpgc01<8WUzj>9Z69hMktT^V&sAxq4e6??`i zS0PJ`wXTeV`3R|PJ20bC-9~{Nvc$kyje44 zM7fpg@$<68c;b!anhmSWcteL@;mw+6>e{;zbuHS%ICGp)Miz|*TgS)q`SF<3alH$S z+E$gz*}o7b5}Dc@pd~LxPErf$wGF_xXaSjYwx01f2G=)cfqSu5o00wX!2)wFCcCQt z5mmO@!?h$a@^YOqp79Mg*d`z<*Vzy)WKISfEk7<@Ct7t@n`88^Ox%-uS_JDib&b7+ z>y+PfH9j+fitRU}t_Hyxn|l%V=G%LPSU01BjEI1CcUhHrRT^G3`X~*lv;8;R?L2e( z(rA!czq%OCYiy33mL)613-bRz`Qe}cK0E!x$;p2_oBhB4`Oham%;6uuI{m}x-|heS z=hM@ZlmGb3+dti2zy42t`uG13)#eWW*WXY6gT4OGNFb-DfB1h!PPCyHa8b@=F~cK$ z0TGa+z}f9S>|WQbzi!VmPC+1`BxrK~i7xJb>fc~j{=WJBB--D2cgyGELKy)Z%f5`X zG2=@;=4JE8l$J3mkDr}o`KO3g!xHY1P^Un(FE~&QPr6|mfcUd9bB%~n!}zmEps`VE z;Np%({V7UqcuY}hQ(}u9oD&<;@-vh_& ztA2Awk+RhNAri$5H7WIK=j4(UEa5pfWfCakSkkI+sp|JM<0&tgnF=%&*K33UJ>W@T1M3Jp5w$PqMn(27^ zz`ctQZ(5RBYFm5$%QLP2Cg!$l9s|wBN=dT#xB<8K#jkw8lQOnYqFMAq2{l~ z`IJgYF5pcxDOvrxBpIWsBp1)fhKs6uH1&b0i)&fsk4cU4WprW<@(KbW^^LFs00M6hMJz0Z`uHF>( zL}Hr$%RM^y1tC6C+u(w&xGIY=JAyAXk<*k)k_FjRS;_Ov?M`qJ?=3LHC4KJ*^-S*+ zJIi28*PQY&@jY>KV4{RR0}w&aa#~Qg{fXRcs5(U_457&Y7c#xkG+!)eR_}rr`*J99 zmP}9)^qn@l=)&VfJ_u~p#>k4_Gtt6I5@9m>BTep@NXfU#Zce%tbYmLd=7qfH8aY|o zf;|u^i>7I;%WO-hqgZw5PE~3|!$|$`5dBeT`Y-r>!eV4m4fD`N+y`4X?JUC{r*|t% z0bk?y#mb>tu6WEYchi; zTF2oIc8%uFVP2=VVk;~1K(WrE>3ny~^B3f`P!+V;p=IkcgK`>>lDIy2ogRyT>3%+Ap2n38?vU#DQ`;`)m zyzsjgHs1g7)wgQMZsdv8=d4W56PB;f4fOqq(IaoqMhafaBEB8bv3&!%U?Q{3^oo&7 z@{;9i2Xb`R>qlOHM94{`f*j*S%B2jtBw8(-rOcYw3Vy*`%acVwrRM}dQz}*AO)n|YTdj7`nl(5q*b|QqT~r$cGYmep-;NV)e^Oe zl%>%7rr;@4#L=f3345Ipp zFg6Gz{IWwJX}aM9`O%*Ri5ynRK(6Zjt7WSDYGf**8N2!>N-HzZHWjlQ^P|_Fc|Z;6 z>k|^-+u}o+_KKG5fyVP}^Ss?YSRGB%f+=MwsKJup5-+c> z$e$3@;eE;rS)%y)N^U3*1}e0B=r&%oi}D$4%wnz`9AW@1W#N_ht+d{1^vR#FHS~^1 z^yRdM9xpQ05|d|_vnd7`xuXv$=6HeWxJU+~m{1&?fL<(%Y2m|FogaW=RoF87My|MR z1g6mkHNA7`Ez3|6Nu{RV@L7L44kd2)B{IZtGv ze-qwM(MFsR??vI6S=RL!!-4mXhu2oo~r-J)YWtK!BkDx>#XbV(d7PKv9vYgU%X~HGJ@KTn;DoD5V6^3 zAw3K3z92b@dUA8z@wDfqFXjL-)ZZ#evnEa0EtPb1S@JTyVRVJlnL}&eQJaGTbVF-? z7ip=hInpGlVc>b(U;h7>57@BxYlGdc8uKB+jnSL3g(wLq;3WPcsg<0MmEt~&wpJ<; zWb(IIw5r*NN6Y zdrLHxE_VmSzwJLbpF|=Y`$SyqGuL=Ma3NoB!;%guvNM%%H!XGZQIS;Bcm^Dq#6fq( zOm@Z2wEk9(U`Yo6lQ;=LpOrJ1V=0Zsxki}Gr`%%KR||9!rtwdp{{lQ5Z9h6BkH@$W zCro~)b@5Ta5HWxf@$h=eF;5#Q-2cgSgcypIJM6#QhpwMNW}+$M%A6ZM!i2X#kTm_x zr^6zwS9&(~xxnMhMhA?x_h%YO7nq25)azTc%588hf*OWHi_=b1V=O5RRIDaQ;^HG{ zCEnFEt1F*3EBiPuJrZ*j)D_-*mHNmjPr7+z_9=_Qk$+$Gyj#dC!%}3ow$TCLBO@U% z9+H%bLMa)#|1i|mNvSZ6;g@C%l+M+7ahP`nE5$Thn(^qQ+9n}A_UB}K7Pt(!rpyo*U3dP*}7$eBXL{m*aU0NA=(OthYdT~*1Hd>yXehO1xB2f zh@T-yU}eyevUk6|!lM8384gq#Oqg4$6L*0eO`>gf1B*i?<7HsbH}T7DM#Sitn#1@2kJ)X-xO`?_!Y1lf6n zOvz97h3TleI?PH(X=~;}L)3V2AlVAPSF(k2r>wZ%QgtTOJIna@_ViG2 z)1QW=R10wmS;7u322?O*HYz3R0~qY(USuCI4NL)gWmb`N&=fjMDY6A3!*2V@@&{8r z#?tDyPEkV+A+kt^LbgwpdsB4z8N^lX$;Pv4HtL7E1zHt2XCy`Y3{xYdPa`qvIyY3fAQ8FBJ=W1G926!_uZ>HjzBQVTW4=Zfsykeg| z&+Ea^gA|RrMOSSVZuHGwPA@o^L(Kv7X`Nsg)KgcEL_*Yp7}FHn&VaPAez6fW*h-ae z-^yBKc?s_K5{#G1vClTs6o}kvAobpv^FsuZuWa)~$t;1!yzjN+L3Lq`$B4X}BLlhL zN`}nLhg{wBl8_gE$wfaeSy?u#;@7$cu|{~(_6d9`hVwV0&(b3 zOpsvLBX7=Cue0O4+L(fsTt%R_qnn(Z&i7xLFkg(x*mmif1PXrU|9<_` z`A8g}ltQ8E;;c0+u8DE$AJ2?b=h6yI6fw`xfUM*zF|%`_gX`64ATia#cAJLBpAWNs zCPyarlG3X)?JFRBE8Rd=w!^HRgVXd;H6zE6)6ije3h>5skoGeTW&3C~Uc$V7frwXQ zA&-?sLxV7vXF%;Zlbn2S0)BAdhd1tt)LlS&Vy-QG&n0KquF*3#{oWJ@@Y*x853Ytj zUKB6gxX=3Unx|&q)}WmYcKbRnYD9sltk*9j)RQQg)i5J0k~HLo`=v1wE03A!B4XUf zXauMFJf%X>(q1bd?cC`+b=@zmh;!>PH5U&@hVs`a;@x`aO7fU&c1R+)L|0Y+czUzu z6>IqE<$R*>5*QNp6$P5HYE?+sRl5*2Wcyn0jjs0lS93LvuO`9~R}#yK z2oRFY!+$%Gfx~7^OWP7!|Ah#0I>cNcfLnq1$DEMkZ`;_ks84FbFX)|#1}d=kmWjcM z!8lkE^let(da7e4LL~-@oiC?D(Ip81Z_KKZzk*~ql_x63zj=7{9Q`THb0dz%MITml z$aq6NE5y9z%b6nGVmvUJr!W0NO0ckX#cntr9f@%|`vyGcqi*%2o}HA-`C;kPokpcr zlNQH0t>aH8Vin0j45g#Qu_*ks5qU|dHZ-_~UV)zd;Vv%cm`v#K!b{S-j+17{!#+fw zlV->}yq9Lk8&mj6KfGrQJ02Q4A~M(4IO4%qTkl5XB+^d62dCq6tfBrpO$_8A^Om_^ zkOvvuVH2ivVwQNx_d_Ql>0^J5Fgl7`;KWo)!Ct(?paT-8Kd?sQT~Uj+7cbi7)R z8UGebLck)$3b@{v)cf}kj2Tc3QxcwbKp(_dLIs!6Lc>-_7lb}H^aN%jh)8cis%}-P z+6*6MQ7pU{r+%>)Uo!?CT%sKnxy-GHXf$0o3mqC(0;#&U>pvI~SY+d5Wz8TIjy6jG7vmI$aPoDGP391Cd+cq<+GS)>ibSWo2fdyB5b0p6TZwV(c+K= zAl?ypm=88Nr9!;?K?4X4-V#R&GhK?NiPi{*+`1pr8vw4_Iw3csa>%YIX8ddPQwdG= z`AMcRD+AKj>1@sCTLhYy1>@TcCc{2fThqPIPDfimRMOMJukb3FFX!Lw5Z^Wo$kPA~ zo@+cD$W!0HB4AV>d99UqF1uGo_TR!%*hb6+FMy_)ZA!vpqGC; z-`ZM1ji()ou4)QA(xy(B$hDGV&CZS$4$S(clR{0~;NB7baPI)Kh)#cAs#_px|5pD~ zwF)9D=%^~)FoO&9?&q)!)PG0z3V+u?!m6ObR!`8n{r&r?3yzP+&+YC$cYTZPwJmFE zJC6Z8td64Zc7tf8tk3BE4;H-slJ^Wrwo<#SA}`?}|K(@Ww-1){HY0gMiJzHXN`5e~ z!EO}g*~b?P!p}i;Q$N?a@@O5@{JWPzR)aU+M3ze8ZNoNCrnq`6-&3G*l7Yfz3d)Fl zaB*6@fRRAIN$09r9iRNPWh6T9pR%XwI= z$7&EC&7f5}7M2=OF%D&&LZ?8eD$*oGddT`wUc+!EZx;voGAR=HQL)l2m2m1ZF3qdD z(k{26c7V^39|pApZLVOjXlmy_Y(SvzTq>rXg+Onun&-E;hC%Y z=s1S?azRztG1(8r4`3QH__Iq~C1-Daq0w4%rE4&Q#`m3`&3nF0=youOL6$pW%I2{s zm_xMUyIFJB`>}GFwL1h26=D8kT=k$u|Zm-={68-982r9Acli6-J>2dt+kPG|_uczGrZs=Wa`^|^eo2&YBFB^%#N%c$;Ke2zAJ?`@s@HhAVa^q)} zjeOj4SvfLcS2XP}UNC&SKs^33;mG7A`$cP+UYc6G!qZ}@RJf`A^7)CTS;T- zP<+T=`OSCgyuHjV65v6sJEc#PZ`?Lab+5LwPUTg0*V<%SchII}boTQ`9 z+5|V>S^ZuxdN*Hd8FL^6qf0~oStPhWkK=O%zZhI})!y6;JbXg>bG`jc7knP{ec^HY z&7xAH<~BUOJ_?>7fLDJd`{w)S9SEL)y?ON>fB$}BK2Z%UEFNjBmgyYN64I1+m@!`E zDZlB-ho_aXdV*M)t#D1rzm+oGV`pCn28R8OcxZq2pMYa1aKY@bl$&X*-?w>r4Uk!} zJxEM4of$fMT6Q-Cvibdic}IXDxImdBK5=Lpo+jN!cx%O}&9^XRa6MlZWJ=Sd{bJH( z+jCIw+VAZC;{-$CA_Sd_7TuxT8cHOFjh@dxk+LhuvsSC^C?VSlXanygvn|SGT|KKX zdZ#T${mn-QvV=3_PVx+0m+2@l(uSt=R21tLeWpy#CQMJ#8BC)~1js}cw{gub+&mz4Hz zoY({#qk_8w^^hD`e@T92<#?C=E8ksb`f+50AdoSnnQ$B7j`?0_$I}?1c>062RDLpB zuIAK)v%09Ij^XxXGV;a|3_Qf&$Zienp(#ubTQaR6t>^t@X#`Jo0dM69QfAfp{>+q= zuSD5Jb(-D(CIH6+^3#cJHiPMi%3anolPFN(CbwewQvL>ycI3QcslwV)omyMQGI5M# z81+n_V}za5spAmFo}mkaK=AoTPMIDADKs0s*W2P70TS2_|<5qzY}4o43v zQ1|X?ySommB#QT+5>(u)3#*o923=+A8LwG;&BoBy8@*)nhg^k14zi?*q^|wVi}30Y zF&Qq+POYc>R%iN{YXhaVqCFnW{_0GgT!0ofCng}b9sJX=ogLuo3`(@oH;ROu2F|pL zd>;4qgAe#3WFGN1G8|z3@*cTI62Ax^^#1tb0V=-<|C&gr`o!$=n*EpEN!VrG`?+=r zYf1aKty@o*Xc)Skft_lm<og>Zj{}Zzm-n5s$39SRK1-lp}?|n?6`_Mgmhc9-1FBIyP_(^ z7)ueY*+#^sxfHLjtqK(L1)ZJBih-%J>7XTaYk8)lh1NaVpUXxbq62hx zZUxH{>^&KEITS=JN4U8TA(4R6p&T<85dAxE3fwP>+lAd$*hI_Ds4&*9_|Aj7i0wm$ zg>(MP_vS@-`eo#e*e>fFly{SQThGdqulj3S(U^q50QLd;MZd$T_`)z_j4T=z#IN!$ z$_U`LU9~lZCT*(Q+*^Bfs%)1@rIbR{g38zm!C3owH4Ynk8nz6{pQ*vJnJ1`zS1uK4 zVP_7-4jHyD;0$c4bmq*zNKeJ4qTbgE1T6b>t(1$6&aL0UC{9LP(K`SuM>*jiCtvvx}@u)9ruC zBh+xCX}~(}plCg~KWVC1JQO{;Q2Dz0LTsD!Xt$fd_wYLWb_)GL>!1x?#n&CD`5%3w zIUBDXPbZrt+DHDno%-?eU>-3nQZKd?FwX5$RDJiiMprI*dH6HSv_eqhV|V>c&>*f~ z-*94{-*F}Km)9CtXaofkmooWDxXcwOg1qb|FvMI0|Rrm(#uA78G^-yAAs>j@2_L@8JlS{#rAK90(?!cJ@t}?s}bK9DxN@aoO6`v2dV3#yKK=Dc;{N&iNk)YG(J8 zv~rl55_58NB(PqOS@L=rDUNeBimL!GoiGqjRgsn&s4>apE+)s*_eBPJUr2;MDH@|3 z@6Jfkbt@%2q&_pp$y5I5!7f`d{fh{s0Cs@~R!PW>_9aiF zf=173x#Kd6Im`wcoKjmW;^?$iP}>d$)0P2W$0r`s@*h>-+chv|Nb*}~&bu*;)*228 zlo4Q1cDs=kDLOeEx|52WKV8z(g;&YXWFW6{*)xmqkh6Ned)LESni`H9J z1rbaU*z%JOpB$BTL&O?r&|gy(USr=awl#JxncM_4Ar*t)wC$#>8SaPt?+1y0C+1`R z4K){=^&t?#h(LY>CMrAst+~`r%f_i3S=;g+VoD3Ez;az#c60}!vMM0_^QNRf``TDy z`l}#(XHDDM95$9S?J_A7Iz9id6?8m7pV7JzAw8-ysTU=XS6irv>hBP*;pvN&jWYm# z73X{$`wq%Jv*rr|PT*Rpr*OV7^bSFBwe#_zOVahq)gHMkl5MY`c#)b-cCn>h)a(zPHT z;zZV0f?J%8>^EUO+6lA;>eNjMwZwY(+=GFOjWX%+CQy7hRoR|~c%X^2&ZS;|zAN5U z;WeHwaor+;i;IL`syM01n+NivA=X71BOVMI>)6#C1G^b_SXxOP_(8t zWAXyC!L2FI%PGteQKv}HK+yD+$rM!TXnSG65}(rw=vZrd+!;F(F?7ii#P*EdpGUb4 zDthgeMRPZI@OKYIt42r8e^wrwdN>jYsRN>}VOpXG&R%627T^5DamZ^3yT_@6K z&nS(oo)NvXqFKJp_7@-)mN4aivmXoyYL<}08M;z0_Jscx+i!>C=}EBEw{mEhWo-Mp zY+Nd^Oz4Ul^;IuI^duwDiRBT!=Q8Z{EbuPoWX^AZWK7D{wtjPi*nzmm2g*lOVoHR? z-Pv{95L_Ov)&myK1_29a;DCiQ%A8*I8|ktBKZLjfW&xS_#0=x$RN&cD6E}39BRTRa}VPoEw(0S)i4GYp-k(-mm}^ z%3RQoMuUm$AwRA4)lA9~0&vR<1nB@as_ZlhEQp&-If7<#Mou#g<*#XYk9B>Sh=_`9 zbTrROetle2OtX740>mV=ZWG$2Tl4OwRo?k+VC;O`yd>`1tJGUBT1zJ;vqEsOV`~+2 zCWBT-*p()!P;7%sLQl26Y4vbabDUV+1T?Pv!75dJu`$i&B#6YbI^+v;NWwEmJAl!x z+JuyIpsnCZcSHirZf)MJ20(hJ5*NmOrHa3x1&(7yn9wkII@t$c!0UM3XvbvBxN?Ma zixEd?T9^7u@NmJL!@r^y^LELB@36xDZl~BHZbdfMbE#-bh=;yTSU0*^IA-aMj$@WY za&V<$+HDtk+2Dj6vMm&%bUE;~&3Vvq@Y|x3wqq|u`Fw3oPc`RfWdKqy|0m+)IvGpG z2E~%|*;oq@_tIfm?lDI&C9Q_y!h~Sr! zy=HbC$dpebn8+ZTJ0~XDa>J%0vbVxes+kjBg|H-Sw->KsIn`T5)88qWT}+iMI( zncQ2ha1C#gXSM7s_?r3hi&9(ZD!MyyaRd_%P>L|$6fAr)E+$-%3@kgD$ zEo@dcyF=M}<}xGmTbR{D?6>poPsYHwu^vH1nmgCn*|C{xQ$9@C1~u{tbs z*x(zXNF;$2&(N{jno9DaGk&++VK(XKJ5Ti_vQYb2-quM%ex`gJhn5K z!eZ$#x)JdI#4>&o>-pVp!6_W|D(<2bQ^SKtucpexRKAGSW1mpmAB>gs1Gd9b4gP#) z$-u{!J&DVP#WemNUIorSo7Bf~`S`pT-oL!G?7vN>L`y2J?$fxwiZM{_R9KdDG_)(p z(b9$!647kr5Flw*2VKZ)L6FJwxT##z9T*n5$ZXVLY#@F*=}^w6Yd%jvtg$)L^ zws{017}PYNU$ezbo@fa|?BtTPbsd`JH~kyJXH~PbesFXs0UR-<-SOzHB_+3=B6}%O z>{;Ml?6f&as}7UhFz0>etf3T6ju3tjrK7MGO(@=&({#_}rZ2arDVI9)ABI}QU8lRx z6`2IF6`46W)7EY+!FKu8`XDCyw+yY-1aG}v2sH%c^}lY1G_=PbsuhL@&de-{wKDm7 z*Rtv?q~@K^4G9;w+Dt>iVE^22Tc#SHb%Kc5sQn`wnAx^;i=Iqx>{<6g>LH=boHy4! z=xSYJl=5=x;LNdGdrH-FdqfSf0)b!-yH@kdGAjLHyjSaI%-yX`W(*^`9QPyKUU{CCLA!(_#j^c z#A763MM&4Iy15*Ll3!1!aw*5pJtv}b8FNCeP4DE?9^vu}_1C$AnYtQX!&n@l0b3hrpRT+qK;{_YNOrE`-!NZa|aph9@15B?a z^p+^kqCeq}?2O{gLe7{uEC!L9@wYuq~&uQN)@=g=~Co!97F�xQg=`vBFuhKkoUH6OY^a>Juu zBWgajGy<3(Gip9`_h_nj4S~!UgscaIU9W3PFej7NHB7DpZ|jfh={$RK1Q_Sct^;qV z<4;R4CnsVJWeQwqz&~*nXfsPN7UvQUNdn>jF1le9{+Bu! ztO6DmOaIsII*`G$C->hIRVlOou?h|O8W^o?D z%2b-mr5f0^EiMmvT?*#@KV-&ekNT||*wxOA(T?ZgL*vlK`hVt|`Dy6Q2aiV%h~L}( z2T@fqyMj`|NRHoeYzfVxu<*ZnW(lqA_kR>ikTgkzczDS2{~4J74G(ynB13Qva|b*{ zn($A$*S;LT<0tdeP{x0BerNVH1oN3-uH%4G1q^iw|c}GU8C1!?D$;(|5?Od>I|&9I0#e-qHEF-aGDCgX-ixk%J~i$^b0i{p=*#pP?>^}$4WoT^AXyv^2F(XywB5%>)On5w zKm<+(`6&6g{#+nE6#WZsC~Yh@qDCVx8^l!w)Wsj*Y$9*LIt?ZUrSNiGashghHi-|a zbI~t*1r)yelBoPov1yHqeI3fqVzt>&Rkhr%n)NW9SSb(fFfwMkj6$p_W>n!WV#Sv~ zuuDSDAFz9t4%~5VrbP;tkj*H2j7C}uG_Vyn_rnlH_B%~m1q26ej+yNHth|3AA*x$6+ zoDVYA&9s}+AFa6p(^QSwmL`)Q+A{a%iEJZ@Yx;a7pqL<<8?pYMoLnGM4rydURw-Nv z^BEGUW1@P9Ghab|UiT&DzXbU!$n?o33wKEQwHV3WY*d_>cAvgmfG&Ra1RLeAEqlColoBmMmWRZ)7o8F!a!YMUG{f_!Vx z&2-I)z;E};g z<@k;!-!Km^pknGWD8Ded3dHInZZ9f}538r7lWDVCA0k^dp{0(Z!M3zRU9WtjW!3t- z3^k!EXwt7tFSuK!ltNE#O3i?X-0PBpMLPi$YajOaYFlIS&>;j}dBh3Jc3AjtocIn8 zFuxK)P6Js7A1=IK38#-ARp1e|r-^$+#GkRyoemUbv$RWJ&p;cZcC5-7mA{WepAY_X z?{AL}ee!t<;m+ZHP2;U8kZXe=nFO@92V}%5h%r=LuJ#>c+lx1KAPFrY(F~YIK2<(1 z#?p$xM7Hvf7RbL_MuNZuJ^bIlmKn<|EKW2v`+S_g2N|&#cP$6_yIU&@rVk;%@2A+M-SwM2)Ztr-CAV%>-@t z(+WUx3IznrvQGWk7nzvWbv3D$&3Me29fwuL6mLDyoMLsNfnr$ien`kIApQ{byi4)6 z$-=^Oh$Z-yrq*-;!rRLu=#`BA_IqENq#?fVy%L7L***Ny*Iia8C~0bT8a2kM}ALWtO0OhmMIcB_J6C&&L}ng(HENG#icA`{b7^ z8e8H_g3F&cV;+T18Ocr@NzNj{lwBnUhz(|2lNP6ne->p#dpvDzY0~D@_I3VCGcCeN zfHONb*o!xWMNk%*^m}r5esSI8;c`&wxDGBDkW4-U)ah~v4}cn&gaRhB1bM!{Pso(5 z8rUUvNICuXe9ZF93Qvs12?t z!o~=;^&IbrQt&bC53MLdcGZiM<-}22t6|6;^`K*QyPA6E;Wzc4p+6xU{$TFCFLA!Z z78LP*^LL}WkMA^|u8KYjbV)4_FWabwIDNid6eNS|0%cMUbXyu5t$Svkcmt`DcYv`;=T?fu15 z71!D!Rb}i^qb*JwTWkd?ZBelOGy8zL?jsS~@R0h#KtqC=_UQo@q8d|AcBi~j$BKIe>;%m$XP(V<^FlO*rjn5$_7r|-%MAm#PZ=o~lmj=6Sl^w5s z0LBv66>`qTl3-*5l*xOpwWr`BU7zBgdy?-W(VT+!uSj89Uy!Xx-%PZkztSPs3JPsP zv~WA@9Sct#h|wXYm^g<1<~asgXdPZ+aTEChLwmf05b57AwlYB8Az*!m z)1EgdRe?0>^^;n>M%Qt`FdI)`S!9}p=J{grwSJ$?VS}|4?Fme)6(+=>^S5b>-iN_@mnXu68#t~B(sHia$ ztDwJUqS4d6TM7e{X^(7YW@09A!_)sS5zI3R1t&^ZX4%&+oZmC@Wm2gb?OZjc z$O-KLB4$kKuwQr#1;tBQpwZ*UH~Hnepnn9Muh&uFIZpdjIPSnO?x7ZG?aX8LJwJbU zY`66zq-D5k`5^n;IL)hB^sD#!F#5DPc`9V3vQ-TcYt{N8arkZRR1oj2mTF zRNE;onUFI%Oq@vHpJS`%emCs6BL>XOlAuTkClg2*+ZYFSM5Po9b|mE{5qN)4#?1_7 z#&mG*=u~r^u!t7uA<9zqoS}*l%-zpdDLJ0+mF*ilmey%lswPB*IOFRgAxVEmdSFq+ z%F~{>Q^Cr^Kxy@}E`=23aw)qv@4WuzYIq!*u1a3N*4@c6Of51sw8%DKR{ENMWm`1+p5PT8g(MY z7)LQ|4p>4WK*s>*DSw+Z6eiI@w>H1mfm$7rUJ0mg)f z{dT(?et~pCXfF8}Rm9k9TQR%U#F}!(CU#C!lqhW}ijyJ}MfrWBO>ni4wiR*ooWqXo zY`dv7j=>Q~*U|eAd92ZSTKC^fd#c!sW8*!Ykyovi^%}-sP*_^}4m}j_)}=!=nGju} zT2GDR+O$FMCe6%>@1k$zpR2qZ`N+UFtjzq)v$lVskU7XH*-H@b#X;1Ht$STHPiLNR zYncN#p9wUHZ3a&U99ubJy1q2&dtMs+XA6wfi)9EX>Q*v#4hjhM{3Dbn6*21bA#gUm zon&%9Vk$RgzQ}U}IqHG-ky%J>#wM8qY-?^CQpB&oQWFNg<)olzBigA{iX0~e4G($U zlL;(}>vHd0)kC|e^*U~86w}4bl-B81LW;tCI@`xDlwwZ|-o&syRkvv<3)LBD9o6Xi z5t!<+L<)|K_J1_m!iVcElN`2&UGB*g3AiK;k3N-W0AYiC!J-#-fxgurHcLSivI8#i z!kDUWoE}ZusrGEC%tVN&7cU9>RWWGT@H5{W>j(*D;YNi0l0=}KIz=~#-G!SO;Mqxz zmJ3=G948!UKBz5iD?TO6gY9do|RTh9NmGa*4a!?t6rk|m}A4O3bT&t>+}jd z>oR0e+{q?%1zUXy-oi+?@>7X2Sbjtl3aZuF6yo5j#oclVlCO+c7ABirsg+fz0n<8H z)K;dEtpugy6}Da&g|Cq*J0)j)vKD-U&>)i1p-83U(R&+eFc2-^-LpQ59fQF7bG6+{ z++hs6(<9^}Q+hg1xt*8$INOS>r#P-B*3=nIL*fvY`>IrWs^_0oyxs2#7Roz0mI>X1 zcfA&yKqO1IAa{Z20JMPiDlgW%w=_)upp_`%cCD10C>2sthoEuBnN|;NEBS4UVPLW?P z^m)i2EAJ10_P-?gM5R>vq1FyuBSg35ua|^1;E5J$>tY1e!^K$ zhI;PRBtLl8QcL@K^wi=u{twikwbQ8PVQkeXe>v`psI?>bGklroFLq)U?%ihv3_jnz z>NIAL9JJ7uUgl-2z4kXnhco=5mt{s)KFW;GyWL&Fo$R06+2G(jUgz{L3HzDRTJ$5Gm`lk8cCu zh@fX{cZ_@qo-oKyI+a_f!XfM~3VcJ35DR3YPE|jbEt6KLo1 za^~YDZG{ppxW+$;l9kLkskBx^&0pnks8gb<)^0=}8*3iJlj3Q#Bf$pU)l6z7lvnP9 zM3U&*gcdHt^zuR_7A4nK94cG>ZNC5J*heL5pEV~*rf!Trbv@~0&3}-Inv<}z6EmQx zSz>G-YiN97aQYb6?McEbvQ(0oSPF@XSbxVtKE`qd zPD24tdb^#w3n*eSIK3g%2nQm!LIi#*n3}-7lr&6AP#*=A@o#5&g~+U}9;~W8yewlK zg60TI90(7+2IC=JFV5REv@?vG!$|-RNcCR0!3*?678S+F6Jg=R z*Bc|ZSKa%03aCXrGpKbQ#p=|YvMM8u0o0x%4RGxNeWqG8Z9eHe1Drr1IAPR8)YN$D zcZI>~mj!e~UJDy=yrO2dYnugci%NSABf-=UjiLfq9S;M&-P{zX%Fy6mn@#8-zx2o4Q-9U5w6Ro;Apx@HR3#4}u|n2F5$m#hU zYy%G>hIJacB2+8_%P7p_eocCcb_{+U0_Sp^I-zm*+&}h28BknN`1aBhM^prtXq$N5 zh%G;pnXX5oelpM1?Xo_>vDu4bngR{0K9sSGZGP zAIgjH*ZHV)C?(pLndg1kyW(w>KrcS+dpvJqh?b-3UN>pP>Msb?etA66;XQ%NrY096 zf+FWci(~$ObX=+rv(YT=ksRWr6X)h#Ivh%aMD;0aN-~hr=vZ0hzqiUG^VppioYAvs zT|69|9GU^U5iEP8sv4kDNwS<(96}4-0nZj>+){BU>R(@j=LOV>tE-F04fv$v38?Kd zUZFc8GF?;%d(lCD9=ChD*eEIZ1-dhc=(Z>5`dO9E%cC6KV$nl7Qcv@kIP&Ea973bW zj49INgA>Pl`47C6wqkPVYP~{=ph@uZqFRN~wd3QFyK*(rtl5qnZ%8P7ZuYX~JF$*p z5SPN=*p>)L5ow!>5`PCWH+ELXuY2 zb}%303Fr{|CF>9hLDmXfmS&t3B@No=b1mV?j}6ck#Sbgkdqf{5|9CC(&jMXLnC94) z{yU)1a$I!BN_eP^KhQ*^MoB}|)Iw2MGh=F_-v4xIpVQ1cIaWmF(bsq^;gY{{%T&`Z ztV7@7KEOdy8QdCL$;qOyvT8-9oU*fq%|(C1##+OsbAyuyh?sytHZ^oA zo0Oe{t|-c~MVD>tW!tuG+qP}nwr$(CZQHi)N!}N{q&rog(5ap^MqsE$=EjEV7p6sy zvC|LKY)27Ff^!H^n*jXe_}Qu}LhjUWSfMZW5tUM!c&ZK~x~AM$d>6VGffX<(qk5Zc;^jqdhAG2Omb$hJ5{}Ngoa>m39pDSZ$ZwoZxA5O zs=Fw+g{Sic8qEphDcb7z^DNU^F%tZ_gCur!G!J>UuVk>PiLmSdZ+SxS@Pf8%i%*gF zUQo$WaQ;Q)Sq(MV2^aen+U>y{wQ;brf9XfoUXB~Vupy&CTvCRxZ7V;5h>ZzLP_SHS zsQ~WZ?D`%FAXH{chW!_B>Tq=&@onQZSmg$)<2ly_vA`_?fZ@iPO$D@45tewDUrmG9JaSy z5+axZQ07fDT$sm0$LdY98{p7LP-UMk2atcx?Lf4lOzF^Lm^#QXSNKTgPI;|ZqP^Y1 z4y9x-k(H^;M3xpmU6nE=;l#a28LR0V2}}wVuD_ZR<;FE_@ax&3=JNdra~QI@Cx~Zf z6KV(M%>Ylij)_y$-nk!}RpB@@6+}Gtdud&(OvEA?ITMLk8)h-SD_)WfxbK8?225)j zHMIT}G?C09=4K|b6!rs&?xs7q|4m&3nS;al6~H?RK%B5ewKuL5;E!4J4t^1-@DSj; z-fgt>j3H{LYqIR!v!9H`@reY345@pxGit(9v;JFX`W)i;`jlX;3{;c{BH0o~L?msz zP1Yc_fN_Z%t=zejl$>%8^NSc;*i##|I^`x~kX~5I=|tj=A%ulw*PiJcMkOPBk%;CB zkGjn;e&u8$WR@8f>w56}el#Loi0(C*Wftx}5%oU6zK3CpdUqcpA%Vy1G1=X{ILfqq zO5YiO?JfHUf$ZFUfdj=I3yzBZ=p#f}ZQvlDYof_;{8<*^BY@!(=Ga3SzSA@j9*I; zV@+RG>AO%$=|Q#X_1J9u>5O*tN$fbp*!2ca1xc}-=@oFWf#7SS>>yGTH%v}wSUG5ulbKa;& zaRAj@`al=%ODsLtMcv|oR>FRs#MRmGjmD327P$JxROkVAun%X zN8rWhxQ6aDqLd<%{&tcsyNNc3;qihqO(xtjfc3gYct=`WS{zEBqaeKqy=$m5;f+(J z$8nyhH9Xigo`HW@fBAj}srt;%rQ~(H2gU<;Yj?L`<>2{S$Y<9%DUxM$b$f6vzajOO z@NfmBpl!zm=c+Br*7gXb@61|xq6@fD4YN+Xr8@mc=}N1n9uwc2J*zW|7^++<+j)SB zX(}p9+kQucr}JY;45w4S(A5A#F}_n!_#t3~*t!rq5!8v!`$K@O)6x58q|>8g2TPzw z;E+9mW68r~%>Fr(80zFPIE~s7Ca=lhIkEw8X9olEG(>oTnB{?M;A+q8*=Gm84{CY8 z@=Gw_NcoGL-s$%!G^C9vq@4U;ONXmNNPW^ivlQ3t@RKi&y~9)6U0c8eroRXi^AY72 zUJs|^Jlst8)hX#bud|p;TH^lwZ!>dCukG$#7#&(`M7Ge;(F#2>ggyrUOsGH=zo+E1 z9b1%L5dCrmU!CME!`6pbKI7>=buXax>WaSwesWdY;pL8I$+PAS4E(lHEQFpN{MubP zKB+l>YzN~azg~L}03Z*?H>>szN;c_wTm(+yWXM7y;V+725~%DEH*PjTAFWQ+Ylb{? zXk|La9CkL9=)hxk(+p8#2_hw4gkVf{+2-6<~+-v%~zM1_H0C)A?D(&K29|D z>WfWiql^ualhcn=d%pe&jPKedX82%#*gTJaY22TF-B;I~-UTkDpU|L2??q#d5TFGc z+HQLPEJMDh%?4&rJ1iD}{A+>)LjKSJP@K+}&Oc?+oLS3IExccU%U2b}0nl-G)zKl8 z4R?~n9U+>^s@Mt!Td{gaEwG+5aBvq2ZQNc`m4n6*RIz8Ta>qbYDPV87;pvPO^WFmX zyEeZJ@GR<%fN^;TBtb5@vzs}E7r@{7M)}qF56N+4l>Eyb?!e0vOIn`=t+UMr9|Vo+ z&h@ET0{HagYT5h)F7}saVO|T*3*I?D>jk-@#wF&c%T@9G|H%2qqkWz$5c^k!tiSzb}N()W2a^XiM zcEIfRR~}-MhdHvbEd2C~fA+-qtiUmdZ06L`7hcK0H1V3w|5zTkN7%K|ZQM>VKlBXd z%)o$c;J-I$u2Kz}muYYPDtn}fZsa=N^2S~Nt2HoxPCaMD$Z@SOfx~t26mARb6eWie zu`}!wE-D>;Sg7!Tqg`e?zo}brPj9)el)uT=7280xU(KJNo6osLp5M9vyA`@I`#j4I zBF`mu=2tNIo#E!ft^_HW5177fZv2J>ZRBI_W0>Oi863)p02Fal}a&@P|OVq69cn}U} zx-X&IQ(+Boj|qoiO8MzUhP*M34m_ajKF103GiBbWm!@sUNJK{tcwTug9#Iz3oErPH zz@MSQXs--JE)KRCjY@(`r((-h#(#vTPKg?ziXOQX~42xMNF<2HU6k2-XTX^VOc*-q_hoR@+PINM` z^PH{exZc)&J{D;5^3JvhBk#73xI58z|KZWWlaalx`TWbuoxP6Mt3c1KYlx^@n0xxo zNk581cJNj}#JTgBg|csnh`we^y(WG>%EtOC*Ww-Vb@nka>x%Cvy z$gbQ>OdSX9((qEao$;?mMx@UO{!ead<8upT9Qe3#Jwm(3r>_H1CbyavC_tL(A zVWPA=>W}ySl-^`NvYj4hUu$Gv$7*~^duZpEYx}rj5tVvDI$N13- zDr-5wts8xWLXs-oR4=vOWOHJtRCb5x(jNI9$9}HKs?w)#$!rMP8P``;efghF@K~BP zDve8c1E~M*I}2`u^zeZc1I!Z_R(d5b0n}zuL5StvHc0=yB|kTp-o;g<|29$G0%eD$ zb&Dc@E0mYiZV-iYd3yHHS9DX%Y8OZIKUesu%){bOfybyk^>fE5h&M1e(!zyr25Pyp z1=ULpZEhrB+qd&(wj*U6WaCZb_+|x!s44i6mWSrzR13+|jn7iL)pbXAmPiRx=SjJ+ zZK`Ue95`Z#) z+72DO)1_0s*>;1U%RhBf{N!I0qed^!tD!A()HOFXdwbJ-`!)Su5o^Pq0$1_@rcwZ3 zQ$g2MLrY>#lj{M5u&$~ZE&rhUA~<>#W8*>-U;obR(Lh%ke2NUo(t`dKrl-q==Yq;S zf~`O6%ho5qJmai}RQx%cIq6hi`d@zJmY19b3Ff#RhC}r&Su5MkY4uW$+Eg$<5%a)s zgJ3VcslLS@em)(6bt5SX`0YlC$45}bj}Av)T)kkGi;2q@*JkMdESR;LtR#)3vd^}O z1bI~U{wbDIaO;7L-tu^DEVZTR@nZLc)Z{To@cyBtq5y%L2{qPV1cu}$r~kkUv)~qc zVd*2fmwBj5eZRMxz$&h=f+y+mb zmjetCrpgUjG?RMT&c#11@Qs5$)cq?JQ9*I4?uO}nU==&d^ofYQntpn+y<9(s!uCma zW`0B03%fXyjnTDHOz~ORu-eu|dp3N^ca%H~2&|c(C?(r(*iq7PVcvlZXI={4t;@=o z6qpD2%37|QZnl7+i-t<)%b#-_%>==F1Jw7nE_5B}s8jOd`uPVb=L}RytvOdsAswyX zV7hiAt`lR9FowYlQrS6GyBT*>kSiW&-mVy$q$@n=nje>de%k-BDB>6k$HEq3;I9Sp zOY>@p8kYO>PnXL=}8*7qjrZV0gFa0Z8YyA8Kr-+bN`+jehz3)~VM z;%+7-Y_4W%R7uG}(Cf002xFE90S;3K5zMj|jdEQp#13dzNQ&#rT*b^Tyn>pC&9IdE zWbIedy)I3^f_YD+Q6dSociN?89C#!%t(vbclJ}-iM_Lbw;E9;`Sx&>V$7XR~viK&= zowG70HB+RugYr>(4}RL(HXaL^9u2`bV8AHFyUxm^b^xP6WS|xVCT=*hPm70r>;?;= zb3G%Q|KmYc{3VoEdZbWWRK)KAoph(A3I>`Mfsy|sFa!PI_I+&cK>Df-67@O5#6baI&oBy~H~y)PcI7m+)2g1dE6_73d{9~qr)nLl0=WZ6VDezIy>ot&`(^FfEFL>oLf3OX z4qQJ!I0roQNuNJ&G$_lSe$}ZeE=uG!7ch&9pQeuGOoIK}bj4(sGPg$n9Afbw_%+h` z`6s);Z=_iCk;8CZD54E8I`bc(TGnc<8aaPxN|@f8(h_~#kQg`a3QgBTK*rrK>@5w} z{Z7HEXb|zDi-K$6sM}dN#h&QGxb+xmbz@_vnP;L>*LmUbdT`784wu1j=bJlPQq>(} zP~GmFxWKtu-|6!$8zdwfU<)h^xJLI=N7R=fcMD=Apju-dP_FK+5|t#~^=Yuh zII&m_G)1?SQx}xde;@BnXKH7W^YzklwzMjOjDNL9DQr#37)Q2%1JjN> zLye{B9;P`l4Y0jA)__TXlOV(zXZh~v!9yd?hP9>fLM31Z3|=!nPyI`p1d0IId9G&o zZCuRytVM>JY0zmz+vTX6X<6T#s6-hRFyq@vG$k>~_mEgh*(-RF{=iwrN;2F6%M-7V zEMzNvq>7L7+NSz;)1ne+D@A5~^ERy`{i@d2PsH$134y(GYfd$dRA;WmeTW9UtlPM}+O=>VVi6y4y;q!heS3*-fw`Ug28{W9T+Ovg~ zN-70K6dryQUcWag7p{6xbgnmCI_iLUqdq*au9e1jh4yG|Uqe3yy?{7*)(Nw046Hf> z$m1)B@P4_Fu-P2oqX7VVhv0Fei7B;iuqoMfwA=esCq-^C?VIyp4i23{AyX^`y$dE=c>vkN%ZCG_UNo;SWS_rUI`Z8fP_+j6@FR1 zV#S``$r|9>5LCWrTXAp6bTGDEFbhWxWtxqUD~fs0p$-5Wb%ZjM)Ad##n;0kA&HtF4 z1Ttts^mj+xY2au&Q#?)k$w#yH z_uFb#Wa9TNmN94M0^g7PMmH zU0h!Ed6tikk^p5Z!$$=STE~YMX5rbyW+jW-3igNDBVj+nyqP|o(qP%qBdQ2f1 z!CGTH2*UY*_5^GgMY9wijN~;2t+>2P@OS6RD?3t(uP_-Ypv`j10fQ5EMzIv-5S$?5#76I%+duB#MC3mjVcdn6*r9uSt~G6TURy0hu*uSD zGN?Z3&bguNZ);#Jt0cOiA5^)PP~*dI@GOQ%U=!a7(6c? z=Z+%bE(nyj2<RHBMw@^sF=?fY~GUwU^)&SU>ww!K$Xl-4r}l%l}+i8!uC6X*pm? zHq)R`}Z-T5`!3SRbQgg2itY@-_~hp<$QD zL}57hgZzJD(BUquI-+_|4NPZSa=O!upuvAK{N$r(c|C9BgI) z%zxQ8pII}TIpfoJ$Vu9Sd$tLNmlKvAk2Y=1)G`P<$R^{5T&i zCIaRG)z-e=IeDK9HG%9P-`)AxPT&9jT=)JAOMx_cqfZH%;*v_57_gNL<`YfIDcV)L^d-8GRdE@dve@qg`62>ZPq&p3b-c{s&qR zLGpqtLiLspFlA=b1cxtf;Mw@u&E0aaHgC2-?QTh2&biS;zgQvD z`C@mY55p(LOI4{u@ReG8Pt~f!uj7Lh3~i;#%P~1(BcB=d-!D&{f=n;)){9HFzlpZ& zE~WJwxUSzK$lDbUaYD|FgV#CPDgfv))KnayYeO~MXZtaQRO)kRy2Fyh)ZAWpOF?Hg_1KxueI4H7qNIK6lKD&wp%j`o9b=jf zM;BqK95F|SmFs4al7>5lOCozh7yL|)9-vF-$A`~0H#WC&%jtc$X-N5o#r(pI*A)gn zv7jVsIJi^Yv4jw>gd_{hroMc6ig@o5cWS10fMY(E-aCoZC{D6vBrU?5KPFo27@?e0 zB@N>_xk`yhvmh_cm$hP^L7Y)f~x# zg7I|xh8B_4T*>Drma|ue_0E#9q~V==dfq?dyGMx0*)oHu4(nlXj&9eb>T4fyKUgi< zl(R)}smGkfN`bhZxb4}0)0e|q;2mnpvK%Jsbb~lG#`7GKUN-cAF2qR?Q~=l6TN`nx zcuBFg?QYI)mW`<^1@T)rL!6M?Bh}Q4cC`n870NeYkO8L0XNlk*cyWsIY}Q>OW^r7Kt$M7<6U|-oAYf2zL%+J`u1IZ&179zVr400)81v~C#=~?o}epBXG3v-y{ zWPGj0*e}vflpqNPrJ2Ue%4_CR3@2P#WUSm7k+mi+=sUjcvr>B$+2_5b9phiHni(`x zPX;d=;=>!sz=CU}q`$J%Q-z=Xz=smZhGDw*X&L(spnKqDG>jM=Hf0}yQ=^5j@#^oO z_XX49B>)oqo{P)%WFgbu&a6mj3JpTINkV7krc6=QHu7SG1qF0;jo3gy&D1|Dzh9;W zL2JdI0FqI^r#R_JU=av$4{SV4mLo#TqhBvS7d~oAwWEwvZ#A_ra-A0mcWFOgKQNw8 zFITkxC56_$JpPQ%d8~0BzV@jjX=hQn-c)iYW<1dZ8n8;Tw6`B$w%wZHJ)N2HTR^%t zzx(-WW`^_8tJBpK@b{WS0E&4;l#MEUcDVu>I3US5vYiTeSA8SCqQQ+FzAP`N$fUHX zIoQ-qckCdPM6k*rlx`X0_?v~Gzg znBC+?Fe-Fv(u0h8+|7F?U2hvA0D?g37B`r&H%e08ULzRGUfp6hP<_Y)dNIufV;CsB z*`PXSSM%mOxjzK6VRi-jRB=k@|K={z&nh*a2-{!W9iK9@Ek$Xzbk7Rm&qaos%W>A1AyzoJA~HVo$=BBbRWguEJeqOF z!CTylIKSoW4U2XQ&#?EX)%nv|b53R5{g0)(VbO^CBbOk2#8oh}2rmDra1u6f<8$y86#Ks2 zZ#4?-a#&jq$|js8PSk^wj^tEZ}OPER(kO z+ABCUVeJd<6wrMI?@o7eh;;+IQXq;GnbPUDofPm@6&^Oped6lQ`CKHlVUzgaS?34H zy@#`_wJa*2U!atTQz8(sCf6Nx3E>CGu}3f2W6h}ivJljdqlfSHY+1Ec(HXelAK0|1<4J+2iS>7u86%7&!Oex zXB_u=RK085+yu$GokdqG6vw;tQ=3gqfy6|DyxBC#l4w3vU{_RfGv9&c?!+0ZWW^Vd z*)11kZV#MrcgHe^w%t>(TB5EF>ytjy3MP8wrLwJP7uLUp7@l}$}n$2 zsElwMw36BLQR*>haUMvQ(^6%%m|+diC0$WnVslbsD-N$jl@^{zgjmCp7ez@VAw?#q zWqyXmXZ3d7ofhZ1HmsxMb5TR>Z2Uy5{XSD*&Ky_gYxP#6+vHhslt8J#c-mw&^=F_! z7i9xqi`D-185mwG`b<)IthpxeW|?E_fc40`DICMs*C^TmyvF_+5XUUdB%*2GaVrL_ zbL=wi^v_jt65$WJGV2}F=^u5~yxGOV^TvTyhTFQxO9FVC@z?##E7+L3*~@P4fP%H@ zh_%bQBkwP#9k)pAM&53y?GB2(N5J$Z7X;9eW_-&K1IK0(-ruHWyahuoPY-r%>89tE zovs3{09GV(O>xMU0DI<_P71a60pP7i)x)}8fv`U*r3Yttcm3RGC1*9%%wtD>|7ZcM z`a0VfUzX5*X~hQ`lXkqRv_-PzrcwbX>=6`bABhSuR>$$1y?p7WGhy(AH~<6D`?3+X zUIS7}o!$-QiabPmx)PUf#Y^Un3gAu)xos~H^&B~~Ogjb)C-hscB=i>)MzH(h#9e$w zZH97xPxEs%Rqe>B99=*y_N}(iGq*^>3?u8pbkRRUqkvj@SlP4*+qyIvDd@8E)A+d1 zNPnCJ6is)_Gs{j2zCjWoO+p}%AV1zHjYN84PyB{rJ*;a5fji|!VQ~R3KLWzr0YGID ziwq?r0+<5Co6JLu*NCol;{oVjkK$yhQXrT3z=}vv4@}Jt+^8wonhI6&m0|c!5<_U0 zAY#A#3J%To;Mpd!cNdy~>#r>^SofH&ieyXhaQ={Vm0?CL_82;$*0usGT9S>7<*#!k)B64$Pbl1lp)tPlNDsg6p!e6A>{j9`5lZ{GbC z8AON^t68jZZ>!=_D@U`clW`8#rFLK;Z~@iJww*U>m_J@!Geg%&6lMJ< z2q247(|)c?Za8X;Gy(E4-%ibqlS})V@C$UB1=_4>a;+jyZlz|oSRxS9E=ifvkV3aZ z=Jm=`bt*fS<6f;9XZRZ?PfyXr=6UmO_si0DQs>8`3N~TtH)9M2@CA}rXMLqzSQ4P9 zW}h<_O61b;BoGokUN)8{qjDDMKJWpp7d~{LUgp(u@Kpjy?9`K@I))@tFbMhD@@3Jm z-L4U$zq7-fcp_E1_{(u_66=-Klh8_(MJ>D(I;wVr(t@1Jkryl_SUybs;*4<2t^J)t zVxYKl((-z$snqCXpI#Bh>BEUm@1GNLwLx*YA55x=nW_m9jR~l-SZI5C=YFo?mpk$c z7L89~x=slzNuJ#CJv&fyotT-np{)y0j`937s!`*b^#=)_^qcGO2WlTQYM5R8($=aK&H=I3|?BvjW=50i5unlI3Rmzr5o*v;dRhVK{0DmkKE=C$w1L`18`1 z5yD_-mqkN~SCXjzTnrf>MqaymfQ&h+>GhZYGUS4xjbhPel#G@WM3#jCWB{{Q4Qjr> zXN+SPe!3{j?wP9{Aa!+S7y@U>6-{kWCe{;|teOlULGe)zc${VJRJv68{?*tWUy?r- zM)VD~(;lzPZg>^bCa%oi3TQx+;#N>b$A70wPhCm`eX1)hk9;3z(i8VDFOE~{YW1l? z40(>rBau`f_0MoN8Stq`(`gK@>A-Lx7SN_AS$S)(Th@qC(<-wQyu5i*r-Ih>C~7o> zq8l~P;L^~5CioNpzj#I4$;4Bo7S@X4)+kV_(cV89JOM^_flcftt^kc15O zOhy=}x9j?VEMdsL+8vC2>{GZ%c@<_|Mb z;rSLOad{j;W;v8c>8EZ?? z@Gaa*Lh9|zdIfMzV7@uI@~VU;lEY;)-^(QG$L87Tl?zHs3j&vQP?rU~cQimxZAviI zkQ`R5$NsJT(=D>5KorsEnqdVMhob};pLHh3C~U~S5gZZkM+r0@`e~;CU1cW%-8u== zf)ruxV~NOj^&kfP%GUJT5B_y#Q?C50l&VNZfz!cxFQ;%}`6%BusFISd!?n#l=jqe) z7v&cCaBcT`|Bo4`hwW{1CCVAj?zF{Tip1`;VrHgV2%f2Ro@uJUcy`3-uAyp^S+9Wz zTIfaLVr>o@tjgPudDK_!|Grl|Qr&V*YU2MK7C2p4Lik}c|L!T81i*euq+Tlpn2hxx zi`EGqt_i(xD-UZAX&{}|me{yWHcfsTDqA&Qd9L-1Va_wZDT~7f=#Bx^r_xXm+S1e) zrgKM%S?;nO4FpYvG`b!eSF!)V4hZpL8Q4Sph6sd2u*OJ5$8c(}HMNYu<<&KyKJ_`i za|?^aLm1~){@}OJ)*M3v!Gl`&#=*fiV2_q#;27y-d94FzRq)eJ~j|s0oER)myQJKu170()BE?BFE-!?1lv}X@enQ}YD}DrNmXx(7pgHmGH zZjH$=anGEJC3z1mlhTJkE3QsA=S{lzCx{?7+H~u^6HNHmq{C8e&+7bqUp=a35N0N$ zXh|2Bpwu@=egk{3^JjDD|u7ApX*9W1Y&5Ct{^vUR<44eYiN zF;u4HdO^tcuyqvSTk9Y{jyteLsoYc0gBUblTu>(Jzcm?|A-sJD&^Mv$MB$Vb-oaIUO8*iw@m-M#y?ZdTBym4q|#`hwp}lo@%9zHVpf6 zIkmzQP*&_qdGJhc1sT_sT592sya>u@M@y<2z%D(;@U>&o)kQ9- zWZG~}%cvCw<^g3j5{Y6g9^)^|rPH)dt%pBo;6$g}2o2shZ$ATo!Oaj1(29^a=ptiY zKpu!^m2XxY#|WBr=@puLzl~CLS(Y`qe||)c7u-`GD|UTTJ#XC)`=s%M+<;5~i+o4? z=(+TDz->DmQ5A5VGohf=%+ltBB_k~EVqO^9nl>H1pc{$p71fS#1T3-G8ZSwuhooCC zcvyo(L>exrQi$XjC{4u-EMo;LDI7)y4K14AoJHMMJ9kn)nnaY&8{TU>yK_P=)iAKP zO_`Er6DG5-k7!8vA<^;P7n8qcz&20kGgkx#=?B!Y5nW$$+I>d{n=Jm6uhkUqn-%hp zDD11{mj$6DPiAO4B&gw|+O?buikb=ei~5g}K<7A_0L%H#oM_rtBR~(DMw6ot6$BG_ zgw+BMKLz0gDb$*tBJ^E#fRFeI_kp8i3}sJh-w0}l1=Ai$4VENA_k~g$us=&d(pajL z6N#?l%7B%#B}>mBsSlS7J|vUiyKu#@MKDSRHYUiMtwKxz(r0j&Tnz!Gmbjq<=jS-V zWx^=aJ4CAFnusjqlzI7^_fcum^5DJBrLbx;wULA+Wy<|CLJG>{XR1lq?=9ZO&0;znDryW{z;asrNok^Ucr_lrwkP77^?L3xZ3#WdFk=6_*oIL-(H-iGS#{P@gKz7H}W9qczWs! zD|ShWg{%A5!KN|Yq6rYQw(ACoe?^nFV~h<|SsQQe`5wUEzCsXNX$#5;WFEpil$k3U zAzgbF+%+%qG#8B&-S4w?K#>WD2EW1O<;tCnoT>$gk?Bw9DK?CFTZ0P#nf0K+(G7i(N_MT(pH-H&a1_XR#Bqn9%d*m?W zE2sHbtLRXe=xp|@p?G!JB-|P}U_m$;R&3g-Qp>2?QB=8WSBeCC!r`367#2EC421j) z9CbxDEAJZM-i5C)(Xq2E(p;_fSeBg0gQ{8_xO>jzpA~$=d{DEA;0DV@qF`>e?g}joZfbMcfov6CTg0tV5^ z&q^0mzhG*-HA-$59YZkD0}kprjSZxpJe|E=jJ{SoZ`|HQ!DF~#yhlE#Np4P_=qWsYiMR-dns@BQj z1=&bUQ0~y8pQPLt9 z97ai&26bKFTbkqPx3t#LoKu(-PK`-J?S%nr^?m*qBtASI>zz2>9%kbu1LaU2u!+Kz zv|yukPex#0CNMVAzjIHmL--GA(L(E7}aCwlAHsGXL_-*lV%`ozXi zQjX*&eg?gSnJ`d8ni(VUj#*GPW@Ir(6lM-$=?+j}067IevXA(->iGRrR$IKY>sIH! zsGq7{EuTqdpvJzu*ltS2)&}&v#D~z#iSBw4sld88@D(WiGJ0S;cho|nyU|Y5=QAdl z!;qd!KL9j~s$WY+#39A+H}3GA@73s}uW@MlgfHmrZ!o^SLGm>2kuEgOaVDXOu=>x2 zPhIq(n2EtV`78Qf`4&NL4oF=Io4zIe{0c3MFK`bE`8MX$gqCjBF@K88VzXm`ZjZ{_ z*pfb^cWy$@22kZTQv@Bf<23MrJXeI#&gWl` zfvW+N?Pe)13^4?2wH;Gz;Bic`o^f{f&Pb>%uXS>F(Nfwp<k$9fecjxk~%XjZKEVHy+qCGBdee`;E2x*%&qBk}Qi5d>fTZCsHUFd+J%&>&@?Hge!bG=bfu zGX>a_I(4IU$W)l{K=otwMbt|GDrYNFtelh1s{`p3?`=Z4GRNa&``15GX`dTMJYLcR z@d%JILj^fABqM1;xo-OJ@0+>d<3tZJ z*pb#rYg6<6NItpe*a5PA0WaAIp%en9wHtqt#upvLPj3|7vxtl zO)(vg+V_jg)`>|sq*7g)gHtV?>&n(4DyQnRn**f&$(z87I?=zK4B&)!1UH=)m+$jY zj6_N3DLqeG39%%fUxmB!sST2Qu^Lp6i;`a6OxZdc6=O4=XaG;Q1ciu%S(+m-;dr^F zHL}aQ8NiKjlDvjQm$agP%My^*A0*P*BP)r@IK~YrX*MRqX?D}NyHFEKjhVh`zUgSu2!nJuvBhmpxK$!$Tnlo>$CIvuJWqbZdg|x z+4+($vt=rVZ7J^}+ZjlP;}&&{10he6XAVO$-iaxT%XIqA>GTgD z2sbYy#>)nj6pX|Xwdr{KY7Sw7DvQ@EsIDG5!k!8jPtUprL6B|UpN3?66&NQL6Vxh@ z7(82U2#F*TBgPsgRzB`tVhQZQ9z}?qhn(JsUxDD@Yc^_KbmvTaJ!#9RhYfVG0ydm{@rZK;=+s zZd;?r=!uq19$kCfWO}Khtw_{sCD5H}H-EXbSV$@23@R@OS(0x*EXjo_XJxp~Gkn2g zuWu|7D0(}z99}Ho8KkFXvd-{PJ|oUo_eX$Jq9lNn7cI#a`)#jRYdHR9*0&q&WaZ?4 zZYecQ+0KKz$vx3ZGuiqtdRCp7avdC;{Pu8S2`4V-l)CBoA-xr0=c-=<9pJ@1S=?qR zF;9Fdq^5b)Z4((6*|f+c26m@@%9*y(1u21CbW1pTTERTA197JZ7W;8uI#g6*q4><5 zV8e3En;D|b#-$SUOX*IhXlR6hS|}e=e0!eD6YDUBVv^fWdZSW%mDjn8x6I{tTw^lb z75d58J~G7zX{o?XqCP9bEtn#Ob1p_`JS1v(lz z%LgEqyD9#sO|eLbT!Q`}Xuo-YHCDOlZpZ7Ih2wvGPb{Yy=>=WEWfpc$OvS5+UO?v&4V~ua$ zUSg@$qh!d#qDHGon6gN;*B4zjN_EFM?9^WT}&MQN2(4~ z1-(;OkE#hbc_@}K?20w43Gc53D9uB~7R<1dvD6ziOWxQtsE!YuICscegRxAT1BhnX z*PI#|wF;i9L*?GP#h6CLQ3gTcR`Id?1Hdse!MdW9usmQ?NR{%9vk3zggpQtw`=0hg zU8+F99+}e$E$&WewW7P|AfNLCZTiX|nnHSTu6uDO)@2IJ*s7fF zJ_Qii#lFMOwICzg;FFT21Ufm0atnAAJ3RuHb*g?lvPL@B6(;F0aI@lz)0eFQ+4!Oa zC5^tZ-+YhS)s}hcKV>Je)c8yMHkBM-8*JmBtEPS%E0#()%1T!%NB1}|uaD`EQ_5-y zUY7~-6X8&F&d+5qEm92ClBYn`hlIBqvOSOeb0~=cUjw!Zu|}J#%kxTvi;CMWg1^1?^PuKV+aGT@OgI*5cQ6m`Wo* zWC2_RX8uCY%fOoB;@zOFyJtIRHqXg1+f+1~`|cc)99-rLTa@M4yPbhuZG%j5=1H&; z&B`VXkXnn=3*X{*=ES6780G?FWObOV!I6j7cB2^hS`~tP16PwR+>hpV!o zC-toz0l)@~vViap@UuPctWYwm*KAJxQ+c-#YqtjRd<6f$;=1X?Yb`_v}9~@%k=B5Fk!63zm(BFtcZ%l&(op#Uq z)P$@QsPwD0ttrM9RqzpJ6UmcKS2eQe(WDh`7s~zZ={R{qV|s^cTZRfB1h(fA~LUJOMJE0~?L9odYNF-noDH z+i>7rdh`=2%9afcm9z?!2}O^wuAJ{VFtmRo7G=ZfpbzmEfpg$?(uNX*4XBq1&HkmI zJ5nzT8v08=_kdA0B0eA9gM$I-{({AkgolDr6a>aXq2YVXo1+rT9} z-x6k$-Az)ptj325#RJ=4$z&my;La0wY__p6AM_m67|~^A1_y}Inh2bUMbL| zzxNKH4`a2{HDOIUjnUh1YF%M}MFBP0(pceu*`-!{9QmLhIuSqhDwz(H57&M)D+*c} z_&!;E$&$`GG_IBxwE_=#OW&CMSw`x6NWtCiGBX)o+(4jztQrTTCMOG89EUeBW}Z7X z@1oDQ6-Jx;>>$PlCZN2=G*ljp9*sOQ6})~Sf^a(1!w7r@v5ftlzv!N4j#Ej>!LY+L#L{k0X5lD*=Tr^}a4Q~6 z1-Gaz93L}HggS1$j6{tCstbvasbDFhHLN!ROPEBKd*;A*U8Ln)~eeRfkcoe#^EC&#@w(oA#^G!YeGXy_v<3D)}4cwVKab+qZrUef?* zF^!Vl%l0p+VVJ&JY52mzL}loZmr@oWTu@yP6F_jJ(XJuIa!m1(R+m5XqYQGzfU7_0r>}-`g3e2MDEnn zQsV0XQLo&kw>2vwtx7%k$>n}~J3hRcJ@S1Y!k2X|_A&GAYVeh<%wk`jZhdHM^re;g zaeUhy*5!tY4odGvKC5vhZJ)_&Kmm-F9 ztm|j6B(t|QZ%jdGRihtaj5Lf5f4Gl@ZZ{DD%VKYVREsP`BhNHj{Ibw(voM;A)aLCh zJXnyf$k9Yg&@0L}RyIkm&L+rs=rgSZ+~$6a_93^_)&d-sfQ9euHDey#qM_mxW9VW& z4#p>IJU}Ht#AtUK`-2AEbK@?0Kx5EqRRihB7=*V#D`>Q6*Jo2mS%wc$V5Z3TJx~LS z6d$xvZ%#8a+8jtKtSIY9^(2$`kd~N8&gA)4Sy^2?_k?CTH%f}==*%u{W?&fdxEFW? zFpDrS)y!g`{J9)9Hmu>oq?h-)!4z!PEu1GM)>_kyXnnlGiht;wRhHsa@i(@*(i~Nc1$LAiV*VX}iiHFV92Y?rx^vd?pQ-VZp z38T(TIQdAkoT9h=PkXHcRdH^yvA^N}k8FF0&ICY!1RUG8{l{j8ZxnML z`H!1+g#b#dV!N|L6^5YOyDHjk8vQoMdI}{#V4@r<4p}F^+FsxPGbyK(8iPj3#!%JV zA}$$LsQt_P>1cKwo?#KY((?mTAN*PIl2WF2kIC8tn%BJCWS~s1y%#1Ro%Z1FII$eG zva)Yh&quS=p%AN2JZN|&keJkV?Amze#gOT*FL(4$Dn@&tr-F0(vTP0;xaVdP&+ZDC z;O!>P_;YY>(4EqiN}NsIPN~~GQdaYT=zcC_I$2@Tj3==sac2@o;D!(HNA)2#VZ02q zuarrTXn?WJs=Pn#XPZ^@LQ<-5+LfX;8m|g`e)Q3%KU+Ow1QGk5*z6@0i{+*EqMxm)7^=O(oG+>_)R{;vz_^1UrX_qmPQqVJFJ8&xYr0jk@M~`C%vP ziOKoO>jm7$iQ)y^rF{|mCzgfm$X&RZa}Ig?!5{*t6!pU}Q5=iBF9G8F2l6f>MkUYz zo#>;gK;jiOz4n-1MVMiD0ILVxjvY2#A%#Q|lh!Q2O`mQYTnF2h;c6lvnsRnk(8A6b z-Jk@NX)>k8D&UntbF>1l3d@+G_{>Lp>Rh>`OBz}$MVCn(pkig)AlM_K72_ti7O@67 zvtcaoSo1vqXg$;%nUtGX(n)w&Ks^00zH)EHG$kukOAIV=ot>g0Z?kM`bb6_Y)&&$b zPi$dEGxXxsI9k?6#b*&Ed_soRrH-R}nv&??EJ9H@!$SjHi{GXgZftN>yA}NVBDW2UI%QdrK)ltR}xy*mWLR9P!wKW z^qw6~u3bOXXRO#CyAkde#;1H%ywc~%lr2(>3^)`TFpQ)OqB1q=eT$$NBB+#0$SJf+ zK~j#1#b*Wx!q5ES^f2cBiC&6rYx#SZ?ZRv5BRpn({Sw@;VK?ox4@ReRhs(jy;m@vl59WM-{WKJyK z!}i={(Gi`oE|N9w7686tmKcTz75Bp0{N-j?V~FQCQuUb=bN4%X0X=K&?KE!_;qBg^UpwLLR%>cC zQtX1R`k#1cgm<-Kv(fD}8tS;}he>>@w^1RF8cR}eabJ^&8%TJwjBj2;lq`5n9 zYRdXf!1}%(-X2!u#s47F8V0Gyo}`gMPt@`yuQQcfILoOo3@D%>o|yh*xEJx1jck^5 zXa0?ZgArO}W1+J@rz4FVZ^y32y6^@dGctTi@M%TX`@AdrMndeZaD{C?7g{bC#E+TU zeYG-I6wLzZCAtHRK3j7s%tnb-SfJpo_gTSMsCE)$xtUct9~9ckM$Q!p)tf{!lDW)I zuYbc1Ih&xw3OkNnT$i?yKf`0?P2rg~xtS~>#2lnrS&yJA{=(>W+{=%;PfZvLZ8QC}s z5q^wD?j=2C6y1_pU`N#Mquw0dSfh+;6HHG>1j~F{`{aRl4)EiC&`PtqanQ==a)Vb^qv(U9$9rqM^Vs?GcxT_2G40u|^qO0<69=R?#c%(#o#I%z zYxNQste9TqEe7`wM&WM!0R!eU^U(y+gXiadbv^svf-y$)uaUJVTr@5~mGNFDPFRUu zK9^-SV&AG8jvprF>pY-Nx+o8Hq2dtU__k}$HN!CI~2$pqYryxrbD+@_rTC}zypnO;9%rW85SJro>%iqv$X$L<5N3dP!pv%zd z6`nhOc>a?-)y^~Yn?$cKM58tjnvwidV=DIs>-6){N9%5-Jn-{l^Fzt~`_Jab|1{eF zYaq4HG*4fvEn3QF7<1;7UqET$Ds65f*r*aP5d@}H6b67Gk zP9C($?z7C3SrNz3oQQpnuPo6c>iA#!cTOP2TVZxipI$FtzUd1^;8RV(bnMTql@(Du zw4(M`vWT~{uFRjk;iC)8>LjZ|{bj=4EpHo%<*R#`qH2AkdtB z47+Uq?<&hX{~vv3j(2`P`JZ^`w>`ca=r{lUR_HgveCW5mZUKW#yo0zm{p-&Vv&@d< zk}BBiAT0KA?$`QKctl5jTFX=$ddjuK=rnnK94-08Gb5j%^_Byw%) z_tQ9J&4nfsOX@alIuCAO;!Ws>A|q>gbFQ9WfUMoH6&nfcw9n#RO_J(KTnTNl9yJrQ z`1KEH^(4^V6{3v)BB4e%`~~ksf;H@!;JprNVTX9pu$TjGa}(OlrO;^27kOSJteRe& z<9^Sj|G%8NJ)jbKvK%(Gl6V$!R||1lY`=qDrVfZjdxCw~HX>F`S71+Wsnz0c% z1B*V|J_YFBQ_ckTXm3E?O7{18EPmA$0CN4_w`7t|^a6%*EFDxd_QdDe#Alo_rp`?V z4NkW{Q}Q`Y#f6v}tXiRWv=#};*|qE_GF+`1+#C+L9?yzmJ}M;NBoe+3_P!Guz9@~} zPV;y6;+wbpI{8=l5Ijj9jgU+XBAT*h?weTO`rSYQyvO%#F#U54Tw}>$qH>jR-FKHD z>rT@NkiB0R#FJKbdZH6ch$_&Sq$Ilmw%b+PW$csEnsN4GXjt>pAoI;z-Pc^x)|y+( z5&Mn3j+)`k@;pe6vVgGhGD}M_3kvW~>fv1PNfngN($W@wy!DNtMEpEHo6xt9m#Gf& zPL+E0tk4G6nPx)zc=FEUOOQBdU;|nF>ubE<^u&3Lp?zh$Qc{A=L14oJn~#MeOp7E& z>c0DIhwj$)Y0R*Nyl)4|S^C+52dP9>_&QfwnY4-tw>TWYmpfw24UQ9nWFG!Xbj zc=#c7UKL{Mjm=>14vuleyDnMcMqr|ZsTd3n7T<%_KcgiCP?_qtWl*Rqfp^mEDVTyy zSRBHMjm=KOX!qax9)t-daR=(@r_q1<7U}23O*PZ)^?W^?O&#kZZDnn&jkLe9zfU(- zRSsQg?!De@#?ica!C%?t(7d<*Wzeg>e0jBnu38uDYKpE&tBA?t3^PdKj>y@L_gU+cOx<0&&$Xz!dMI^W1z8+s^ zOZ^pks!SzzaiGB-&hRt9)@_1Q?dxD%2dE>jqHMun7>`gU2yDQ!T*EDeWS4aZo$?Zs zD}T3h+2+v=151Soo_WoghnUr<{BkTznz>pgUQl~&F5iE?L4y?|APw0LMdBK*|6Ev# z0Vt}=b3-q`FZZ4`d?pt&KRGJMoo5^Qw$F(oqkEAfC5g0xSwGUV*;$A=9 zsOxP171kwupCOIMTRX@=wO3e4zBQ&vHOB$Oj-sE7po2}vD6}z~*&Lq;ugkDT+x=Ag z!TOR?sGbMzugS1US^6&Eu^TW5A2jlE z^pdsj&vfK!EU6*VacD8xL&&x|7A+LFKDta5BfdiGNmxy6T5EE;9d&Zqssfo9sC^w0 z$_L1>II3gN6n1mbez5Iyhpag`W>0fH)~JwVw<-8DQ|bdT?JaGd9FO^;uSNyfxf@BL z_JpZsdeNQ{p{l2}2O0eC{*6eTKND>&C8<>6w7aG6qr>RN3?>s*JfkoY_@(Foi#G|K zL2xiBT#eNAcMbTsj73#pxVtWCWA1+I_`>t6yeS%+@DT_ghV*m;&A_6Pre=EE8~yAd zbe$Tr9pYCs21RGPZrRKwOp7MI?4OnR0R4ZpR5kUO@zON{vPbV zOYy%R4-&2uhj*4St?(G0a2vshXyP`N?h_`tf!yWwOz9#={g7o~sSvau)(?<@gP7Ez zFwRo<-K#Cj$*bY=Ls;f*lT6LeiJI!_R6^-x#g+wTjc+GVHZFV|Ykr?s!p=&FAQv7y zuI}Y-Nt91BJ=|xk*7SFNYdb8av?;46W?k^UQ?h?#CSj%%3A!G;yosQp*mteL1x);C zMV?8qESHvt&5gxaro!St1SZu-?dBeATUpM&{yUT|^){{J!1yg2<1FIxZPoYT6r}fr zTHj$$V+Iawj%rG!@qDWfhgbllqO25@ z>;=i_MQ9|Im`8fMtY3WmJ!u!7qcP(~a52=S!(upFeCp}m8c(w^)`5`zdCmeXL%xsP zjKi-MV%5#^QQvHNCB-_Uxn5v``?C#Ztd+;e2!QwkGaDJYmEWFs4?-J=gWEWBp%8$;#PmF4-TD!|z|1u%{%W4$$OP9^&NlX0 z!NNF7dgQI4&ufqHplA`LkD)oKCl}eAQQ(Jc0-B#%@T{3)(?xSB6d4DjsLA( z=GVc<7GCAD#y;1%?3Ar7;*mjb|1}!At1%!Ek1%V>gEpXe_yV1BFlNn%>2vcMm+fGW1XU`o6HleNOCQ{&Idt zmyuBJeSO`W)%*8;ywA*GmVYsn`@dg~Lq~QQVUla#0}KE7u`zn&+8p?L-;rxOYc?uf zaEG=?G#g@Fq*|;%(!teTz^U5na-t|swsigMgtxD;aPp!VdWWw(u*U#F03D@7Jk)a` zvKQe4*k;DF+aSZM^zUgwC%H+dw2I{gfna~;77+)Y#4WTVZ6ZLmtlNq7fp>9+@quD% z$cJd+)}rFbRKuF_oi5(mnrAamu20e^u25oTg-74`RDpIOC!;R{a1^_H2rpt1Q=|jL zcId`85}d+Tkx+{R`EW5qQKS_3Lj5p~W6)gFo_7f>mVFRUEr1egHr)hTajW%(qwR5o zU1Q32e!*<#DWb|ajAxM0+Z~w^J0w@V1wiI@``ox14_)V2<{TW>XY|pGT`y#^h$XKZ zGYnVYd=X)bFfy{lh+HZbvyY?clnJfJ&On#91vt&kx$LxBaC9&wiZ-#xxSG-Sec?p_ zUB;~vHSeqcmDbGJ&ZQISICA-`Q*d$}G}?UuU!P@l243{wqS(KW7+ zb;l*nMEG5!=r6`?Thi`Yu@anZwTUDQ5PV)ucm`TPH{RTJgQ=2fB8 zTF0RUWrKvgHBY3B^d{SCzY4zAv04mTdFo)=o(nKYl6_6YZ5>|>TSCz%Iq z-ejLsABy)FnTL(_d)bE#vV6WOA9f0_678A2jzK!*N~_z92BVp1U_5>P=un;VjC~|; z5!-(ZXAXF{1(1?|Fpg{16TPynpbD&pOv(?cjxX6V1f(HOs<$DL!pM}=3q&@V$Y*Hpf!4YMFpX8_KwaMAZffxxD|PoG__G8{)IL$#Hc#DDhtGzXT1l zHe7gVmAK{5u_La)uOi`sli=#Che(v+z?fmRmO& zcK=K$I498v&MqUKpFDz&(o8F@ewyyFi7(%~+QY>0;uA9NTUvN74gyGGz{i7D=k?oS9Y=l1O|uIw!6z9s2;HGp!) zt7D|FOK$CdUPrPK&u(}B_Hlbl?^=y0)hh`!x%3ZLEIx4pgy4#h+H+f_{5=m?tOXm)JTG=5Vq)z3|7VSC25LTpU zqK>Lc(}`}+I|63iIB`cD43b^RVOHPa4#8zxu`N%(1GIR16aIcX@Ft6y#g2>el1R0M zRZeMG*yi@rX)db73^x85*JBj4mX`=cMO(jxg0T^zX~~DKoM&5yu5=)gM^z%`?PJF?r1%CN%Q3@d*1NP)*U1EM)LHZ{}JvUPi8X1a1Q zXe3BVx7;Pz1O+fSPtM`_l)d@CI?3Gs-pSnm&YDO4QYwG_uHVbv|0;j|7LWXHf#{TM zyFDGpq7S~8kk-SOSr;YnF-`y<*tztc3N(8YYQJXzbz9$grzj1ez90-~&OAXTy59%a zO1$c^`_qgxCHmO%78IiQ5}XU-L$cLw1gxz|b?zrEgG1<-do2Gw%?kq+7-t5Uzz2kC zK{!4;(YBg#&yx^&OAp5!q#`>=QM4B&_)M_qu~8&w(eSBLO+7HyOh(QX z?1oF6WN5}poYv<`oH+9(Pl%EyncRGmCz&eo;%l68Fwp__;W|{ZwtWnF*K@_ykr`rF zPTWc4t|RuE$vsN|&ASm6E|uba=b^*l1^tvFlt`>Dx6iv&?Z+nWPH^q$sHG^)667qf z%oZa}b{C2Hqbk(u>{FATov0=u19@qovnc?a*FD8b;bo8)Z;l{;c^0yH0dzN4OUOp;XChcF7OZM9Q8fFgZoZ~UGgryTNV*Wc`P_s4giU1VJynE2L1 z#w;NfGR(?IDNA*Y$S>Qmq=;1M1O_)RKiJYf4vL*Jq*$@fwSPYe4xp!^#F%+X3_3|X zdL@Um;QFX_O*k|Ph4y0rP6JGmqsXDImDGNSKh{KiIG=SI2LrCvhjpdwWjelQtc;UM zmBCRRm~|2W-Rd);64-7H)T?I35qo0ba%apyuFg2N$sn^z8DA%sr{rB~F7QYK?(KQ) zwN;FuhLtFCAdkT|+_dH-vmL<+?hPaweXAe7C5pEkyIdI4#3DnAlhVefRuEdQkG3}= zk>@yRg{1iX@8agGxSQj|sXM3MAXuZ70Y?3nlT=>w-G!=4igoi{_1r~sJj#&O)5}kJ6w#lRSVJWdEC4V42!$o z)BB}RZCiN9LlCWK>xS`^749o`sp-0NtL2M}0j+|E7ELRoVdWyJYXU*-plJgFap23S zKZzEk_)$*)50WG2?5m*ZW-*YNf~%Q8KdlH8L+E3}T2iV7eAokmH=G1y1L@HTmoAGC zz&S}rN5*A4{4Yr*(4G}b&F!pyviY${>?K;`CR=UY227hE3<9(Nyxoc>!f=MIAxizWAXv1~o< z2eytt9(Teyommjq)w zCSe7Y_4VAa2;r#65Q3e;73n?PUJ!D?l-cDxO+ejEUhkFj=!W^3VM5Gcde;hF4h^&J z-VU6e?-w8rUoR5(bH#28xBKHw9nbsP#M0YXeY>8=%|N_wbsaQMV||CPy>S zn@r8WWGM-5icKuD&b7h`lO3XYAyB2CI-DuKvTtMf0q}hde>4(<T;rJ$5af!oF&Y%dvFon$eF>#Ts+gx*14ZF&yT>-B16yr;C{nf?nhZ3xPYNcs9dbJ|G)|Bj`iW>+ z*<`JAmTlzOnEtI9pp4Nm@D_!aTRV%^Q9I|$A zH<2m1{Cw=3Xub*;NaLy%feL=-&Z~Tq`Zlp>+l6!xQ$o<1%v8(v=#Y`LI|!d9)t@N) zik(?4(?3WHR83wTvZ9NeSiZo6N@pbwIb9&-lwwW@`ETX2CIHr}Apg;RZtaV=LfMRX z`?3nCHwb7gXXjDBsDsNwAZBH#b);sXab@?l>L;k&7vQxe)n8rKv%${kq}8~_;8(Lp z+ASlvkeF+RiTntajA5izF+c)8y~FpI+R|!~zPO6Tw;X^y9&8KY29k(pb_iQR(8O67iiX46b(vQb zV7oxMe~PFn=wA~~uk%TxC}OVMMg9Xvd*R0-N!w|M5UZi9T77QsL6Hz zvgqJiubNOu+j+VOlmthu3tr~}jm^5Xy~=@R8RJ`~4s?|=*zqp|Q#)fJZ%CpJbiH~j z16!pIe7;A~fHX7~!p=(1t>+csO50{?#!ZPg|N@5SzNv|Ss z+;DLp3&?GZ6Kv1wOU^@F7ejBEqC9_BDR1F~C=iG&e-w8!##OKm>|K3k5?|BYs8WnO z7}c}2Ff>8rA%(ofC4lWbefd?#)(cuoY-C(_0fO;Hm%DVY~03fmbs!sV=ye*{3=ZIaqhKZwrzG=WjGa z5p%_~=WoFZ1;zbdnF9(xX{(Abj)DZW=8CFr7nb$}v)BpJJqhxxRYJAiV}{D)5y!{Q zEpg8e%8Kq^9W!!oEap57YJgG{qKQYOwp!99MX7$a3S;B1s+gzW$V2VoP+e|h>e1xe zImG==0afa*=FUhz#mtKKci-Y8m*nADh8$v=30TwS_HLq}{V{p+!)Scl% zJn*)d;fc$n-{IYd9PbOC1*6yN$?r2y5!S|UJ%idV$Wm;*^daQ{RS?iv2@5lX7<15= z(ZZqq)WNb_chEr)ON}o|G3F)WnG*R|h$$fu^i!2;E*ooSi8_JP-ZRf!Yb#2a@3?YV zUzV(aoy-(Cqytaj8N-5bU>PtPlhp}#O`J@m!Jn9Mx=TDVZ#=$f{G!_~$AUj2jRSxS z-OeEMmt1RG7EOZ^gVz;kpuT>##)7p!KL=-?u-R@8j==o9T;Egw=1vLyIr;dw-ak2R zdwaV)9ufUM9`~ZT`v&~KeIDFjP6-oZbMk&n2@_^`@$rK(S9DUG|8g=A>skMm64n~d zpt>>F^W*#k1wd3ZORFgsQ(zpQ_DMhZ+#hn~4A;*s*XJ?h{W=QvqL5Dv<6BQJETS*L zR-j4khxq{6+`Z=qjInbj)f7xkr;vVSq1PNs+PlTAs>jeshSXS&Ap8lNshY#p>mC)H ze*JNwdc;RUGg7|rYXGTRc`332Ya`bxan6biN{qhMf3Hp0e*_v6jP5U4q5;2RO9C$G z7N(q%htudaNzgC}js?z0S^%Q&^=BjX8NfKzr>{@&EcntJrO9~4seT_IwtA2;cykNU z)Tv(cS5t-{>wC{Pg^+_g_^^u!slau+l2?nU}ns^shVe%#8QCeT1;G8R&( z8WGvTkXeLoHk*JKwV9f<3M*j5;&>?q85+iCD`Z7gvjh#qw7uk$-H2ToLX=6h3We&e zR!}axBdN1A30^EJ6T=q+B53VEU=fEtG9OYVZ%jM+@%uEI3uUfDdA4w%+C;8Ymgj$M+qSq-e=9kNRW^SEcxyV)k3 zOd$IDhv<@=6kIwZrDdC~RLZGqNm*wFn?i$C4P9a?O;wXJRUfO|L^BFkErG1sbVQ0X z72^)KhFkV5;v1rI3rBW=5(0ZOWeH7gngiaMvAS3-tByvkDzk!IpV=G+djRNnROVT5 zs-C5V;va2tka-^7P6tUR#rTH($E`!Gd#xjeuD#MH0<5B9tbW0bRe%=C_KAp%C*-o~ z0$6J^qd#N0pjYWE=h#O>pcyfys`x*5zP;aJ4ft!}NB}6lE|#hni_^<0GAlxEbZDNx za0Y$KJ}E#=Wv~4&p9;?#jPa@G2F`c~>y$R8M@Tm;z6G+gv*rqjv(=vxiR?7imcwO} zRj_t3(i{XiMf)7IE{CSWj33J}wIGTRim0r=sFrUly{tCAn_lEdPHI7Fkqltk4%p6Hw55q;zwnT(;Wwvq!m7!|fBoM?Ff}L?^8_BLU_y%a*Ly7cY?AM6 zA~6^=D*T@dRKLkR7m*cPGPLGvx2Xm|Er7TZKQLi?prf;Y5qOq;chXkCw)dEF#ORd`l0NRQ^dPn%0DbE@Ce;;r2D|NEytHVHIdz1~X0y58RMwu`Z$; z6=MXy`|l8t;*2oSjU(dP;&1`nIwI<8gN-cRkZql6MQL8F@!QR6!n!S#aG}=w%Wcm< zwF^O89;34;|%aO$s*U$!<@6nJ1X2FM=9+@Y;^Rx!|CJfOsaRXAK@7m zN1-?NEmLHkgNqR*F;3sBqQ9FDOu6@z#@qI6Gjldb<82nP%#Oksqdt-PA-D~~4F_Td zG(X9*BE;7l&nGf{ztNguiP>7_{+J0f1m84g>rx3-_?|XQA=i5r;Cuwb!F;^MM1uX} z$NzBem@bYe@BG(MswJQpH4Vz#?);7o;ytZP6T`(%&}MvGa8tm6T-UaNTmA*Ht} zIjCW5#j;!~-9Q?zkNd5~ zIRChfx<24mPY%ZIgqhZ}li9C9aB8I|N`JMX@RZE?u99_O&Z&Y0bjmZh- zMmsLe!}UAn5@`eoNl9j%b~{I6x0QhpfmZ%=j>-_7!^Gny4Gj(XD0&Y4Ly9=D#3bfU zr&7A5LSj-q>w97poyV%d8@Gmaj*uP5W`h8fRmd!}Bf1lQCD&rXw}B)?BQvBxQ~o^8 zP8Hd@bp#_fw-VQTtV!nk*Kk>lKJ0X6EtCq|q#svUPXvqY52Yn_o3?G#oc!lwSV8iG znz`0LvWRhb&^`RY1dNc>QmRC@mvC%KAZ@Kw|4XJ>Vn507_RTMJI0q|vZHIsi)o(!c zIrY2T;UK$D;e88t%XLJVt*?$ZQ_Elv5vBQtoAI-?WgZjV5Yk9hJ3%f}A5CZqJlW6@ zg8!m6WsmEkcK4jWY0pZM4ML6)DY|Q{&8i9F-y*dNp7BN(G-7fK*p?O|n5Ub#8w~J6 zxqBnqnteibTXVc0Ktb96qCAmXIXR<&O^ytG%`l@1zD>bW07& zmwU|w=`b2G_bxJYNqDG{KrFaVMHfy}^pi{*&)pTq^jMq4xwTQ80ciU2kuAS;H9}`- z@lT{|#Onrgo~mX~;zewhV*y_ML`1j*NQdsdZF|c?m-l+AxoySg7vnh5%cp>4w~hPC z_=jpXk*;KnSXW%BGU4wkc)Ao5zp2pT3#ZowriNCzdO?9Ra#Gyh-HcO|u2SfFM%EG; zV{{jptO8X!?R#VsM2A8CqU5%X`>`|>rYMd$;8;oX?ax(0oLyfd(2?-IsMEgV$C&f$ zKaeA@FZuMop3#Y0Z&RgewlYB65YuB12k7fP7gZ}TC7tiZ@&1q-htxry)7fTcwju9J zcjQFI@K3|bhj}-Hw-@w2^WD3B@L5ZLzSvne37vFyv|Apiob<4Vi8d=K6kV&dj&kEt zXVqmxXuJ2!cn?(Dn!(-X{cbq}48#bn2A!4dh3Rg#mrds(p3QD{r69kJsB6#Sqf zqO%q$HbDZ=0uV0vA@6QEEed~7VcA%%9{4m!AfM(dC_ouSiwhz?<^-eh!)Pw}7v@Ea zG0I?_k3*PoR?NVz61#y?x+kEpO2)8H31}B9%)XJN8W|>OdjT1AL=wSb-4Opg_?#T<|nh zKw;5qOGJXjMcX(rh8ll;XkrfyiNiNh-f4=@F|e!$Zv6Qb!ibGr>1)-r#N(;AUe+mA zMO;ynfmjZujJM2FA)1mP87i8I0dq`ZIfF{(4FP+IY?h{1&XsGNMH}>X@FG$=dC4uu z?KM|qJ`qagadD42tnaHW*f)L~V<-!5q@Z+=<@MxY@G6+hhT7;Gt0!vX9ZX9`w=V&W zVrMv+%iPm9G2M_2@CgOxi@bAP4$PY)3r6O{ocWqX+_KrxblIh?$i7FlC1WLQakS6=L1QX zv{u=nLjaq1GIi~V@059)ktwz`L|V7ug^1FC!)-<&d2AC69D^mpWJ(e&ts}f13kJGW zxx}y3QQ4*HwCJQdx7XF8Q{N1u3H?_znXuVYvRWg?|$hfKoC%wI{fgq9KN>q zGjR`E`KZ8bS#P~57aOPRz8iqcFAe-e_wa5~aI$C$^zUOPa%+gpxXLE*B1yvBU!}fC zfWcZO#`MQQOTu1+N&PLm?Qk`8lk7AsbBF9WfAJ!JFMWt{ zWXqfB>gk3@qHp!t$g#aWBzI|mz!XK`3XOgOIk zWN-U51ec8FfMi(|YslqwW7!*U;ZY=SskmYW43`*o9&V9esWIg4t`2ql7l3=0&lPFO znKv48isBASp<)Bk4zPw{YWBFP&y!NCBvBocBM#G2(ZSMdV1JU+#gCwHsE`;7)c1cv zYiy978DTXde{j;v>5Dem%Lai7+D}uVC!|I4*1+rC(0TD$8h@>1Q%56zNg1m40q($#lmR|bKQ*zir) zvj(?fRi)>m(ZirV<0OMehojJ7RIJ#CQ}u{1$wiDZ=0TMr!9CHP;%T@8?|2lR9?@qs zU!o=r6(YvA)gl7&s%<^jN5dRVxU)aSjJf_Fo;g7H_RP`V6x>^mJFk6mE1Y=R$5_7FCKXf01h z3&cXo$V4DY;I$xV)S9Y}uhNjU=si`HfJTRX!Rs)X{~2lNhsN0ltgF=G{8e8)zOyKV zcMWe?UNOOYreA5`R*+VkU1O8qP`;FSJNYMJt%qS)`2#M$#?vBT&Fnqow)k+tt|W#P zIv7wF;UtT`6{Lou)!0h4!Y^+^(&oh=xX5YSMmGPz)3p?;6dOUpL_xr)5vbkH-MY*N zRN&t6FVP(M^+qvHkQ2SATiyb1`KF<3|F=q0+pZ7V9#QsfTdA;1LK{9lA2~}ESFxTW zLkABxCr3HANvz@^+<+WJEy7YfLvSrh-KDz1X=3O_n32`JWJ5F=z3Klx=LIT)E|dX^ z7n$J2cb=0EwG{%ML8M!iW2Ts$Ug$_|GfRw*jpfn8im&MArC%KD3mL%}9e2p(3y8X_ zkS_M2Ho*a@HWre}QK>p=ynrXSz1*=|ZY)TRlp{So| z3P$;)F&Zh9J|(q8$KSD`@SkW2hJ7G9>lzh|xKqSr*dzTRuBfb_bP&XYVL<&LwjT!r zMg4ssd@gX6iT(Nmg14Yr-~xRhbT`0pd?@>C8Vw8q4^f;D@eogJAlFRXiXTg4lKg%_ zPvsJE1j$%@BqyFB_`LKho+eq38EAXR#<9{m1eq;glA}17@92pl*ogwz_+go)G(OYZ~? zxPa?|I0j(-Pk`E^GJT*vsxY09l$Zw_)V&MTh#g6QJWmo5gdWl$Zzh4w(2D>F71f+X z)dEMKjOva=YQz^t2Y9i-4KTLYZEw{5)qlVpRw#jLaiBF!C%y&wSmD&33uK)^Rj-E> zM3L8!Bv!CkV>|+?AovYZ;+{PARUpN|WxZBQ%w@gQt~5vw^8UatFF|FLFGyV9Bgx-K z@+scQ>F$?BsDaMQffmRj$9@>(LI7uzY`O?=`zj#~{w}(ON(hw6gN;5#24si@F^()o zTL6_|=uwN!#enKx2mwSTZTpz-l>VZj!FISJf&){EF+c@RG8rHgZNY|+a1tgIW#xy9I}YQEvUq4F zhi@!JF78i201|@94jv088e#dl-xoIyjL$NS;3PAgFJ7m8vCxQSVid#RRTP;bgEDM^ zlto*)7}}2>6~c@SpohW-v%@O#(_gUVCW;6%3@y^e4MFyd+LBU33n4pN4At9q! z$UpakAp!$Qvhc$Yyb?H{*B74-22_p&0>nUpT=#tsYZCjI-{g-^c#xjXfco8%PjnEL<7t>+044i)?X9?#n^y#o0V>3;^(x(a+B z7JL@z&~1UI!YQMiH{qA2!YQKcD1h2$fykre^uCGEb`p9FspAKFjR`)1w(}lop2JK! zTN=2>!*gMYLJcMMKnU1_uGTCraEEJK&iBCI$#^n4n z=d9f%a^H~NQ*j$ay&$mgJ4Y9rM4>9nvO{|+Iof@AXv&f(wBNgjQWMr-($HgP}iCBe~>0y9X-IC>M)B2Udd34<@KI#q_NLlDX3 za1qb_1A63tl(jshqNF%VgO6`9gk@uqXePA{Q<)Kd@CT2}9j}nrM5kd?p(gj3{~wwU z)a`M5je@$BV6P!}9>=g4_^|&(6b3b&@`J1ZLO{L0^I1%?p}NY`u7+zdT)dRA{gPWB z;B+gg_Kokbd_3L64h@)D{HDxUWAM>Fay*EaxZX+=8V~_Rm(ywz-iem2IM~I;yfQ}W zCa>xpEpjaXxWgN0ha0~)RxHqMuC0{?R$SU)3rFrO0%Qpia#4gWIFA9zMT!4W5spZQ zt&S%x3A&F0@wN?Au1eE1i>FF+6XZh+aqEVr5Q#vG3Gki8d6MD;ug?O!0V6zkIx0rF zPM9Rr0sG)e>=CW(^EFUs*H}RBDA1Y2Gu-St zyjZPmRnkPQGiz;NGBBfy6vVk`B&yc<7LB=Ou4*6Ug1WGxXZLW!GJkoRZ@k>ty_cP; z)VjB9#mblpy|%C{9;-@1!74C4bS;}h<#aCMF_z|7H(++idr7YRq!ka#h!Uv%k^r7H zk@PuRNtQywkNV7jPwTrId;4T?r=HMYo}V2oq4;0q^K1X4cyT+ehx>EPbA+|;WS3U= zR(?l@e269co16 z+!gtyOam=sy4wo02H3;4p9Rn^wXI<`U}0_87!=mzQqrYq&yBg)2eZ*<&;-E?Z!@md z-v=JJ5(Sp3M3xg)d|*&NeHuKW&GXWH9j2u@919J5C1-e<&!M!l-i+aA`A`^F?J60~ zUHs?X={C(Dq=%IT%2fyGqGEq(j{WXQP_P`hR5_`tWFqz|VJzWRs>8X?G|&sFG&62sxURfVi}8_o{7E><6fh+wJ8&E2zh!2!^mQ$F^1F+R7;iid;%p-dLhzv#U|giM5N&9 zNfb8pCLbjVj;FycHr?XLA5+&VGKY4D~s6T2tNPDp<7_ zenkc^`6f|OQ#ZwEqRs=(Pt?9Dw;p#L{`nDqzk|M*c`hQZVhMnT`qGVJDp#E@;+WBS z!FZUhh&*LwWMY=+J}F6ILT)~KkI=(LAWe3F)w27@w3hW3S;>Ir9odD;mO z!{Nv#q6|zF!Te)_o~OD1YY3VMz&_G%9UIs2-=WPAzQEyO9WTP*u&)6QuYl3*BF;hZ zJnzROkJ$Qm`vJ{8B_l5A3E_Fg%8J6Un{G`w7H)2YiEMw*hg;QTSgC^fwh9yCQA+#+ z;+`KXvp$v&^iz~zL%F??=4KA>Qv{W@cJjPQ3-Lq{MKn2YYUY~%YZ=RU44qjL5sfzm zA@QO>d4w85mFpsBRVRyW2Dbf(Myc!}BMx8lCW#?}$l*e?Zhia5f#n!eA{H-)nf?O0 zSuu(xI_t-#n;2$m24ZZ%8>%{j5UPJ-&IDBhM^3+tNj2=|S}n*G1PX{u<2FM)&8!g~ z4lNE`FqTT4ieS}tcrPmUlDubk`kt`J*g1k`vo?WiRl~(s7fjZ3;2<{|~S~7Bc44yTf(=|Ij z2AF(CQ%rBxzZ}}3imnakv(ot_0e`MOs_;1+-ho@{Stz_W0E-lu%~ktmlgC66eXohv ztn(!+0p=np>2~m#V~3ixG+C5@7ODv2w?($oB$pRf$!1egcxZiIEWZ5X;u#RFg53>S zj*Oj%7uCOi=~$j&XF;{0g%}Vkx?vS|Rkm*ZI$^*x(9uDzpgkga$`U$g<1$Vk$r9sO zX7^iGa)=|yAU!sxJL7DUd|mJrPpG0q;8_xbF}SHdi>k>&R6J?KsFG!j$|28>Ut(vp=T)SH;C0v&|_uy3x zwWh0&z*bcK zN4VrDO(1R*LgAacpNaL_r9yeV7TvRD7NAELduJ~5fCa_qMbSRWxJb~EE6=HLs{}$njQIAU(SoStXNl@m*lzUwJ81wRY#C$vFnd=zmFG3@fBve)m_^*lWY%bZCZFQX$lOMXoMi z{}nrx>Sm?28~lB_vlu)MWuUMMF$S5)s$|rql3Rj%?P@W>uceF1n z3WES+!A9Hx(1W4|G=Xx5)rRCO7bT+RHOvY|B-|)N=M_fDBqGnTHg6@!4P#rhNa6xF zkOg_2ReYPl4nBYJ%yk`j&Lr81x+Iw>)=D2odZu_sS%YqmWUP~PGET+ z`Nk1}Vg+72``N@xobKA#iPJml=hXUnWBr_4KTG@f^7JpK!yy)dd$nfOE0Flk^^_y- z_}|1G5vz(7SfE*#YX`luJ+~b>a(H(?`!%`ZITQ7OX0f*4u)XH75@|KlXnZ&afvwE-eS<$N7)!R!YJkp$1tnAGa!k1lAO)DxJ z94>ZtaSj{@Eq$+ytq9{5)mzsI{fcMo+6Crvov^aPH-O?_->@D0)2>X{s1`1&8I)-g z8VO19O0}~Bi>)pc&Z`{;{lY!HWhGxh?2~t?pwG4`4?13)zXGrOI~u5h)SZg?#wdFb zT|p}fc6!MoGs3hfaP&p+Ak+$aP zUmn#IZmdHiSNl>|Ai3KxUE7LTM_ZIiqmP)5Iza{oHWp_+M337ix@=7Vm-N%4GqR*) z4pGHC-Q|Z50wm<~ArhvpD~`G$84!WDAmm*fBJYwlE-~HG4ckp*XYcu)ksW;(kmf{U z4FB$FPDJx4BSuPS7qr?KAj(%~AY~ahaaXTI`?k-XHj;)#96B|cSV=PjL<>twgc9KB zK15+N6_CUOFV2-T%8H679+i(w0%gQJCV?_Ey5AjTp9IQDpqvECiR?@Q<>yaxVgyQU zR6IbWOj@ywta!%uWlO(QrK2hyQ>f&HWJz7J%NxG^9_){i^LouCq!HmDI-(}fqJShB zb8@t~_V9(qavYz~VZ9$m2c{e>;G;MN-=^Py3wt*;rl{ z!KotGEsjHt=T0#Nts*&799If+BtJKb;zWmVA$0!uBe*;3`yJL@(f%V$mld*tIPAxx zVS)Lof*ETyz>CpVp@Fl`SEN8BwC=cJCtRa0 zMCn>m@I7l{)~f&pVV$+J^bew7RZLb*2-meBK4gJ%?g9PiYD88F1At@$3*tE=CFB`kqK5|Y^Hz20uIsV1;$f6%4dnE=i99r@ z5D4)lE9#7Thr#fIeE90s7hinwCv$S4oCN&M1aRuAwYT6l@Mn>$?`93e;{184ntQSN z@{hSenrs@a3$9ZNaycW?Wnjr?WHTe_jNHwLL)H)M844+;m|t`(lf8j^ThL8PsRXIF znV>E~)Q;V)gh{tHf!T_o3}_C9rHH2$-|ByviY$vfed-ob1}3aDXD6pOn9v8uJbCrO zLv=xa^5z&lFhZB{>wllC?)S)gU&K0v^}PdiJ^yZp^t`Z+=hyGtx?NDO8>`bfF6Sdt zq!o7}g*U|NmnKVBRw2egN;I%Ehl!hzsTCh9{lR+Oz%AsR_-T_Z?@u_;swznA5O1v{ zBTE%gWKKx}zpWtZ(vMK?6na{uPK_~uY49NT7#xcziN~>A;Yi$!avw%tPo0LhK~rff z0ZY3cl$=G_n|MJwXusI(%tC|il(cO!Ej>++MJ-i>f4wVc&JzRXdxq$~evDwd*yEyi zVW9wGURO$oY>-1B^x8|eC94cD7|v6A@KA%0ioHu~EpS`JPy+ZJE2?q_3C|)`JZRB* z+w1F%_4LMgLHDy4fz`EHR|Xux9&?~gLbf7Z44g6B#B~RdJFR3FY_kQlR>Qk7P$f+% zwAO&Y0fL;+V3Sd2o5Y)%V|<3*q(c&Grj0@H>acx9G=TR54!*~N>>b^HlQ@u^ZXMqs zEp>M|77#cH1{@Cr91Q~eSponb4F22I6y74Q!(T30SF)hFs0*+PBqHC|-s(0M>NOU> z9<6V5#KWgKE$Esxr4Ho0CD2bp`hoZI7*;=>$Q0jJW?W^g(*J#a}x z&Z|n7F`z9=@I})g915fuD~Va(ApnV>l}7K#c*3h}XZtrm1HR+l8unI|E_OnGh zdW}5B?i1#f_;7%q=YeSmrCzr23gmq?3vCmobsyt*av>n+On5*#c-MWP(ar=lV2ac!s0JCTPR#VjyUv4bRZa>!L!d$ByAZ$l7;G zJ%}*{e{a2Rpy~FY{49(*`+e8yTft48=3D8j0YKbt8*m;V{Ej9&&FHw`W4zIG<68+? zvf@G2tllj3ixSYebnR}c-vWdpk2R2n4Qw%{Oz8T8R-0(UD@0b~;0!*ajzgmcE-w<{ zxQ$hSQK)N<7k(p!lqN~6a~mSRiSaP*?*+W(NG4C{$A36N5`KHh<9U|ZdS zsn23!G9jL#UA_VV4N*|J3lw3brKpQk2Lcr>(rdePO*nN`KlHWe`{)eF}8?NYl<5PiK`T(*;W(uhW@RMy_=t89O z*#1^gt`<3euOX*2vuC8(iWJ4|tEO?#ZK{eU8x*~Cw}61}P#!5SDqY!0(fdY&1JuMn z->}^bp;0l~{y(fk3h%pixe=LIm_VEMyy}WT?s-*0ucu?(ZrBc!Au4TH;}TJI-A;(+@R$tP zH}_mt@_(41jeN-q9GtF3CfZ(|jq2*)fZ3d7IIKe#*$gmhrJD>0t7HIG{U1vm*((aU z;ai<9>7RfJ<256|2wq$tnA@9w)impJc>I;KUHviEbLO`+Q%tQAN|#F$TB7qv<0DI< zBWV=%akiMyq9Hx)NsuT)ZfRNZq|Rtb)ID8`(rFhBXF)Z`^pYi_NaNJNtNvI0Vh&qY za*>*VarepZUX&A!uddImSQ)z0TT8p=b{V{^D^aprR+5ucsBfH2vRj@gnd1MjQs_bjYYb7>*UT6Ks*+J%&E^?7fzRCI z%yX7vX_ixi@h*3=V$Ql5{yGSHL4)3DFJySvfMDR8+ZDsG|K3s1tkg?0&wCakF7-`B zQ&vBnAVf0F!6N@u=Z}(-t>KR0?K$vXiiC{+IAlhD~%-8 zX0*&6jmg7wt+Z>u-T6OCsQ$JvaeAItE;qDvk^UgT?o1+TMLU~dOevtE^CE3zP<347 z0)%Z3Wu>iAgG(HX_Uefe_-LZd43%gtOrzK7 z{$rs(%|jKK>|q5#qGCo5ObjFff6VI07}w}^-sh+|800}Y-Kv}Q{~Dv&f08Zb?ouZB zoDs>3b;dg1Y1_Bmg=)Q{qlaw3v>*%vE!C*{{9mbpLNbkBgzBcZ#iJXxJDTm?&ZG0< zXx8}7I=9X6S#Pu8%sGPZjt&(kB2HMuE922&+U@zqlX%b#wRqS0#t1K!79m@XE!&VY zuK@z&PKc62)gBckh(OUQctW$JRzi#p<4}!zt&~Q>&V;zBw`NZp&nYi39Ht#FsI0ANGj1+a2DgMvA#&pmy(B1@BT^nlGiZE2&jbC&^v^*291{3hq(7Q``UR;Ru zs-q&i_ub0A)AfxZqmQNJ#Qd;q>}{vcJh}jDk6<)mM(D~^@{+3i%NuqVMWc@2;mf@g zg_NyJhQ*BFdck$eJ=}W9NL{d7rnWuhQdb=|F%#G7|3d%&3-UAhnUD)|LG1sl|B;``TrIMmnr}6q&$@)v z%Gf1RpK)Nb*d>shM0nf{YRWJEhxr!j{8vO2x`pQ_Fojcc2bRN}-k??bEU9XmsllFn zF@NrkMi51}nOiOtqck%;d49U+OdX0Cwo`PrYo^|rHH?}TJLu9)4-6?dD|UBu7bFNq zZdvnNMmBuCVWk~y3;uAp$K2%CVrOm&#;XN6ua1u;7}QZ~(uoMh!`7B`d=6*(nRd6h zB`lpm=n8VTd>UW8IM}C=H#ay^EN$MmJm_?QN*RLUoL_1P#}6CUFQeV3M!F27FOYy( z{sfeZJCv%O=L|Bh4Ywx&b`O3jNj@HR>BYw*@KiMljSjKYVk1V)I@>hYLYLA6DXB=? zh~WdMa%X*d>%jh1(@ZMTzXm5;xwj&ICnFx*E|{VyEc&t_XL#@>co@|@ysC(2pPhSV zXo8#O-+|z2-Xpny%zD*~CPWvCn}F|J>9fu8J9lqXb7c#j%}l_R_MDKVjtisUVY2Uq zUUJVdRlEibiH#!6%r%wWZF;6J2)r0_-NHD3=+no>oYj$=K#Km#!$iHze1K`cm|3gf z8>xQ(k5N3n<>h{c_fj+ENG6T{_sV1KFî>`btdkPT`hDf;0QVsC{?9tZay=Vj! z$*UR&$cz;14!ELP9PfP^STvm!YctcCk{h6&yUrMP6FsRm)@X8HoP0s%JlmfLQ64Ga&`JV95RqQMU&dntE6uV|m29&yv^5(mqhbncC!;=e< zva{8S7rfeCh>Rz3ky=OZ_r9}u^hVr~JBG^+N6$I`3-GAhgLta`(VQ*dQw^)rQmbJ( zA(33dHh;Ut5Jy_Fmt;=y2$A%k+4|oqlxlMDS%ajMip1z+*TXP`g1_fQ77<%_C5&6R_yrr@6)g)fd6k8@CR* z?-HvMc3wbquPE?{`L5Y2qpqB^2n&Z=*((U)7<&oquY6Ykx^UA03?K0x_rYWBJ3Q8L z$CIO|yQnMO+l+Sa3G zo!qd>c##lHd|Ov!Ni$j`>Z0_cY^q_*Vf8@g*dmMy^RfvwmNMwlib3Uo(VH-zVn?ls zo@G+Z6uB`C?;GQw3;fxw)qt|1nIdDAPSv+OUwYqrjGq?n{pl0d1!0RdxYT<`>!_Rd zZ^{!KxfMoORdw?4CK{`w28FR!r$}c?ptL&lX$(irm)#L5aq?(}z5b+A=&W8n!f0nS z=hcwq=Ft1RXP=5Qrthn5UCsruW>nu(x7S0_eZB&89>HXBeB4;C@H+P6Bb7MR(F>g| zpBV9%R4mIhVzyDsLH(>mtABH7xoeqo@5excc4OLsX%g)XNaBnvYd3XU^IOFTqYQj# z4BfD7OG=j3&cH^kZ~?bJv#kLL5~(R8IqGcW5yC3DFL{S)^A*rHlf1}oS-B%yAtjIw zAn{a6zJ^S%fP0cws6oQ)jIwp4wXGwt;)o7;eoT9SWyJT}twM@F`QqvGr_c8kd}pJ! z;(r1EwN`oa|MS&=ox~H_py$-NDUpVIuTS9x*wMc@ZSDHxi>EK1K0i@&esQX1Z&~JF z%%7hU4>)M2MLjbnJGS}r!Xz$>GSxBUdJ~<^s+nOS55?DW!*AjGSvU4h0J#B|0zg1p#G~Xxf18*G~3-3pd>hg*=|NG+2hx2!D z-d~-)eMzqDPumhGvcXs3g%x8y#Md5%cVYl51{&VSI@#Y-e~c>@PrFjkL2xP!~nnWOwz{k-1*~#J2~mjr#161u(UrB4+uU3oJU?WD8%9(1|FQS> z-;LW=-uV99zXE6aJ;yVxC3~jXrkk_d9>?~iemXB+%SpN?>1H4jl2DTbOMv#MPoDq% zdvGs6f)WXWk~}XB`9tCnNf5XI?!CBgpRP2LArXtcit1*brwP4+SVVJ(gPlaB?UdF$ zM8rLTQ6SKlH`nKS-vC&bEj8-H1nG>ovp|;tvwgK7F``?>R;mFru)s_ZKd0N$95Q23 zH_~Na%?7FUAf7E>7BuQEQ{pLz8d3^1pa|*)^rTHnEX177dBWA2Q1EeMSv^)+^r8RV z#u`(xj!?2l1jRyjO@m!B34sxqbNkxB(qSQMY<=?x5V#QmsUx_Xe;w z=nf(<>JA}@V`rjHfoHsjP)HGvoPXp$i-=Z>xSBQ(~E_$26j z$JVnRY1Sjj+Q!wd6!-Q$OwRSnxJQz2JIJ;`Zrd-@K4p1!V3?u-bPJPEJ3DppQl*kF}QZe-!g@qJGF$JvA25Fi%`t19r3mtOs&qr4Z+%K;AwTXH~?p> ziKW%qVh_eve=@5_tvnd6R&Ou*@U$KUOKWc&t^RXG4}R7oVP_p0H>-F00`RhS#LDto zGj_zu@}<7)h>_(*@Y@9+3n!?1>F$D!)sKI*U;c^*^Qsf?YR9A$9Dbi}gG?z@tFeu? z9p!L!ic;mJYUrL|et-t1)#tX*L|au8fkqa?TP4S!19yq%>M#mbEFz#op&a`g7DRZH5{XC9aDT zvyy&IA0)%!&h>EhZPsP-3$gkxWOuex8@cg$TM7WqU;u^#``TJy0$^T&(rZzY2Z}i} zz-gxBmgirQ%S@J7P)lumrq?70DWO&3(7ZO3nIjkkUL$-Dxx^gP9^da1_FjGQRnMiNHvnau*yjAo5hm!EZ}(vrZcRV1i# z71;%|_sG6sF|vx%$YExyShD_XStX%Z-9WkY3_`Pu<3V>+RLx5??t2j%g=2 z0Vg+?XBV$8r)m7_m3WvaF)1aRaOIPX0oa6rp^YJke!0iJM_SBHYNbi|)&@;ttey#t zO08p42<1P$6x0MxQe%EAlLTq43dW9IOVi1R*CJ}eNn@xzIz{hs4O%?37$6^JQ4Wzd z{uuX7TG#J1sXc4_-gkP;O|bkOTN8i=U5G-j&7!%Mn2LZ1C#7V?6hHl`5VudzL1%AR zN9m(((%!H|8)ePUl|R_ea8Ltlrqxn1w)0@S# zTMm=ntKo}YKdl#_a79#Mb-jYd0-}@(m~p?9a+WiVQ=WY?+@N2_v?|Na`XkTcZmMsX zG~wB%0%?9TPMFHa?5mz_wFi+|FLcuf-MY6n$)B#}KavT4eUisD{XZSYanmU0HxRK$ zGs;;{+#u6jUuUtg5SQ=X^F}^^_V-Di`-9}>9^?$u@pjikYQ79fG!o-Z0#q^NR&EAi zP79hc#flnw(3GImf4idU}f3r(b^gB@qRoBxVs$Hy`vi zj4k~8^XF6YKWDFBLHJ{?Bx0xISM>vphCK+E`uab!w{L0`x#pX2YOKiF2pslz^Y>qT zF(ntg04xJ;fSznbrVSa+gd!9WH9<}CpKVtNt)KvL6Vn9va-|H`j$XY9j})dLBI^`A zcvQqj{b3&}H4p}TOcH*_$U=Toh%&E`PECNUw@s4_iJhp|qGaj=#v%3ECmVYSp5D+1 z4Y15zqcXM+zn_>`4ha3gyppD1jaclIC9>&Sc_ID|OTh0)w%MNcup*7pZ83rzSAj)v z(n^RAC4?Zn4z#LA2(femvm zBse597&KH{cP(JXs!3@CSWGffbNsr$^NL>Osrn<*{s^cm4`agVT#YtzNvNUJ$1CY{ z*91X>YJ0FY4_b&R?Gei@9Xm>mRDQdoRK}}CHY?0pOPDK+U4a|W>_J5`UXLJ{@fsqY zG4D8pu1zLqHoU7)SzZUJf&ncKk;wgF&UhVIF5|VMbjB-GM&lJGsqyOb@Zr_iW7f@w z!#6AyIpd-tS^!0qAStN*mWfQsYr~XrbJ4jJNvW{$kj3!2$OJBf38g8zMBoDJBNKT%fR|$7EfaoeytXR;9J##G#EMKvd6&`$e8w_c4 z-c~e=4JvcQ3ScDcy<&x)i}TqPi3@(u3Q3-R?Mq!ZwN^Sk|3T&n_)>VFGCXcsbLMQh zy4yHP9Ae)QwX(2G^?HK@O<;v>ZOSyJF$2P`3dWcMaH;Q)#S;;41F}bi&J-Haud1c2re;Xrx_*rUnW^ z^E~l7--BqKiI`oN^MuRQY_QS4?PUzUs*db^IMk%_HniCOCCelSy=0@}j-G}-`fP3( z0XcmMGj)3Otv7Z5zw1x&)ViX6 zpjI(O1aLOoi6TiVF!?kiBC`z~6PhIUJ9WJPe{TzBHh^#0jn0n}XPX6B zsH=^;%@}K?0Q3R;(NnhtIxM0RYS95dZsKC1X`z2+yh}GAK?3@@t6Ce2hwOnC@!54} z{#a8btDkq3Qu{N}ztg!WmCa%BIJ~yKV;|EC)@EQcmFG7-S>7el>gDp8?}+S?Vmmn1 z&{F1Yb#$tYPPNa&sdj}J$>h&qwE= zvX3y_FatFK^=c73i^13yWDiUovZEWZNB?kjjG*mZyAO{mOKj79*ibD(c45Erv8iB$ zv>K#>G}VlNW<%VEj&m@?c@*G2bTo?~=WK}k(9th~*hG*9($F#jn*FTZhmO#^HlD<7 zS7HDWT>CcA8H+<0b$?pd>(yJ$coIYQYSC~X9vw}mM;Q1~9~ertnEZjTED~|3mst zT?XdUN~QxZHfc#AG$f8I6VdjwdvZ?;**{`3T+NF4J(nVAxoC;FejBh@3Fs+9m|XLn z;Lf+=!f;c6-&9*g&TRHaAo)#^&3bQa=)BMUrE%=#KsNmN+n*;j+``izr|`#bp8e_B z&-Nd`efI3>(;xow`kUM9%fInwKmU-G=^g&-=chlg%fAIS^6c54+JPK4uZG6QGc^&# z1WErD)T#?;#1d$i{Zb35iiuTP`Xem$Au4un@z!P!9$mu@jbHh2(J8$Rfy?ZI2c|IE zr<;ijilwx+34gYmHc~|iIA@x2G7FN3h$aE$ z@<>yQfyHxa_o{US>LsMVh}s{RMHA^ac+)&jcmxIU;iL-FTrHbhsOkHjr)5e?sTaA( z%ro9r&5o%$_+V@KLva+IL4(cnd!J4)t0-BUf*_;R9xr7HhYd3V)PKD#iUqj$D|J&= z716ggW5$#4-DnF~SCdd?CJ^m|Y@bh3*V$3S=@)J`;w?zJJ*ggViZ`b+v=bE9Ih^^L zP#ecpaewP5zO9TkM(joF$W(6OLDk~18?nK_C$EFY|kr~b9N&viML~|N(<-u_*XmqDv1)m*s@0di- zOn9?9ZOs%PwJJop?AG6C^>&m{X{p=Vt~n)360bAbpSp{2b&?C?{egZEO0KKY%jj=9 zPbg2xQ^^>)v`ZTA%>3#yS|f{QteB|aqcW@o<}l?xpSvl*uPM;l$!FEFYQ5dP914iT zve*JFi-i-{adv$r-P1ho8B?IzJNQIoOSj%uy+gG)$OzB$IXo_Df`R}$-`V8(>mfeSHPp>B@sN!Wlb(aMS&?lk03jn` z!$vd%WfF|ydTM1ix+7^Bqh1J3q44Tw6p50ih@#g4tr(`i-`DHsmZf<@Rd_7NlkXj? zx#NA+dzxnPgeH2~U4M7p7b#Hf)J7=Qt!1lOg?gkFwLyE7-~wRjoW*gCJ%gr{hl~|= zS6qL0eoXk3P3^zDP14 zJ1ubv{XhUjh#o4t-d&4$ma}MT@Oj?h!I-d26+IQUu0`yCmt&s`_IDdS0aI4l zZ6m5!)*G;Fvtcu#436~z+*6t+Pr+J<#Se~&5(4-R@Ir5!XBlDdxr9n6NL!d1ioxcU zf}%gESAs2k<6o%4ETP|ILj9&%`ZkCVG!8-Nz2FwtO0$3r4E8-FFta>glaxzE@7R>g zw8G(rF5paN1ZwoNn3S2~{^%q9X7zm_gR7)mEr={5Efn*DO{_Z9xA@x_swoA*ZFfDVfkW?P8AG4~a>2W{6xFsP^It33I;HOBL zqA^u;P-OONS|Mkdh!-H36A2TRvss-4M9y7^X5beqOCzYIq^9JJ2=ugKKsKn#fU<@) z%Brs+vt%)eOVGw^X4>Zy47SW_?QRf7Z|lVWsE)D)TD0k!{mx6>Aj=dln5}D|7q}sG znc>22lDywAt$Fs6DOm?+qCC@#(P%Hmas$E+rlGf0YK#@_2Lqf$DU4jw(zkE$LtJ!- z2lraUVg9@g=(hUhI{5;VdnA~*@PQz=Lkg{^O^jQzqW8O7nrrQKQX$PimnQJJ$@37G zNI*oPy#{*PLh@DZrFqmW*e$eC`KNlQAGv?z*E_aam|9hBEM-UI7S(+PV|_N z^VA;kr)ENI?WpGCAr5P@@j=kc+E1w_aVX50Xf=PWi5CbwZ&D`pf-XdoumZ?sPXA%c zpi#vH5LeLkbzGT$s-sHA9)jAm`Wcm2P143CfZ5ANCE9O>IS%W7AwV*;ah9!-m>15X zL4!|Ft4lpBedNjLJkbK(S$tK0n{qFVrow6IhAoVx#{H?;eo(8m>}|Aqr73I)=t{KqM{1gM~f4!Hv$Yui#~SE?mMt zth5hy65DIeG&`s(CIO+{ZwwC6!>hT6{{pxj#_?7`v=>Z&DY!`(=@eiBX|^^&dTPas zc#_jXtScn*g&HJkENSA8V8$Ihyur zjM9bPuojn3bd=#S8&JK2d#W3|5B8?eyPXXT?$|GYnlyRkK6;a8wXzuq6*>aa08STf z5?rP0g8e~%?U?Xx3$1%ml8J0$lJzU+jnTVp1ToqaXcnf~8Kn(+>o$LOePx*R(@bQ@ z_VC#-#SJZKt%neX^@wM&mIojf!H^-#C}M$I$(RjYB;;BI81kf4 zMJpzW6hX_K5S~dzlcdJ#HVZ(2#%X*>t|A)7!kz4UV*$+*YD`p%GK(NyMV^I&vZ*HcQiMe`&S`Y_K#N#Hfdi^IBDNi) z*5)c$+0jwh7Okh5XPbJaJLsqNy|8V%siQutbYICT46vFAVU3nbq*U<;EP#CBc-PVj zrNYKigl$n!ML=0HlV$c~yS5s*?l%ePtS+1yhX!G@RM81bAsT(mbodrSNWha`u)xik z>O&m`7~F-e2~oLBoepnJWSExl8}dE)(4CghyqYB9K@#;KDytW2e__+LFx*HUZyX0q zyIQ@}QyKN)h*vE_bH{qGKd>ax|Mzk19@DpH^n96Nc0ZTaG-;A+p2mg9^H4=euk~k} zqlYk%;+{ze2C_*%K6nQATC$84RDt8uRPf{rLbtlw*a|xn!d(XFWVTZ}kW)L;Zdw3y zqFw=Z^9D{A6oz_6zBLf=m3*o4vODYpVM-9%5YRq=o*QE#n3AVwNxpKv94J^S-c+To zHmTzFJE+C;LP+_>Hrr(eu|o(oFxX*^C-Lzl-Zs4PB;I!Y5>Da{58HXDm3G!NE;@tW zc>|L-LYY7X(5xvLW&d3=iymhNegd1*|s`E>o;Ab}=zj`3pVb$h#y;_hP7KwXS-~zaOpVLfok-elm z3H1osrBXYE0IIGTpD@r1Suu1o-Kimfsr{wfG=ZUc!S1;z!{}hr7)pg2hE(RPzT(RY z+0}+BL+DKrho6Qqq?oEwHfya!5YTWy7k4%HkD+m5GDfJl8eT%K@pgD-EZH$Tmh6@x zOFq@)jOKNKWK^=ndtPBiJN`H~$3mM~ME!X9o0nLC4X;$pNtri1Vepn{R9!qtX-1c9 zQZ+lNlY_kI%kDMMSwX5){BeK$a`(rLE%UIkLr^k@Pi~c0xV=LfMRM);hUnxX+89f;R)bC!MJAjzQWpc4ghe{e&C)gjLl1{+8N<#j!+1&wSwj9%vSLlNUTCvILSuL$kx3q# z+!f=k3poi4Ra3xCclloH#!&5Qr}On@BhBK3K#JfcSCP@xSTR_%p~Hr{)%11t<|6zY zH6V}9ZW*D2n%vV+4J9|ilCTCbgiT11J&51gl#{U= zoWKI%ZYyT8r8XLM5}5)DbAchctwb20!K?t%+J@~Zl9xp;q``gK+%sXElUEguZF(qXEms=^Qbc;T)13XCibdxn{U+rJ{-*BOc-+v9Kw@LMRn6 zUziBOnd1yS#$pnt9)w$jIjC7n-orUj@&7f$i-KojGn%nIj&k=k>WQR$BuUzbuw<_m zTra3@ZwsI`N^J%dkg*r<)K|U(ekvB^Wf+X4(;`oiX=2iVP=seuQpOlRge+9^st`WE z14(z%M(S6s4rudTs?efP2H(bbA+e%bvr$-`g+Zea)VxCu*%O!@&jSJ9>V0Iw(a;M+TeLEYBL;6}Np2S^K)L%nQJ&ZN6Wl?in@!q`~7X2q$>fzi7L` za>-R%`7C+n$Yo|;@V3hv+RfB}Wv;{ni3NFJ5QwDpRLCrlA&5E7=piXD5@}wDl-nqI zyC(K7z?+m_NTlWrCuOFmw?>+Ic}^59mQ3k(NLHqE<}GS^Sd6j3(49vWc4uQ0y8~|$ za${SB3?$GY7ov!4h^K`LMc_C+^WPcSSY}Fkg?1RICzAjB@85)jtsT|xbj{|2D3bVr z$Nfd#BzdYEdgdERfN6IS60F~N`S8Hw0mOjEn?$fjUrhv}sX%zlGNm_n;n&liLTk^+ zCC8F{&^Excq;sCt^&RU8=<@>UePs>(k|``221Or)S`o?9GEp>RqLj(nyPQ7cB=if{ zguTa&5GG`+$D6_y(zbYHC#?6_U;|X@jNAC6lDt*XJocVNrr1iJluO=O8ta4N;n&dK zrGz5i{;713@b?uFyFt`{o@&5tD?B}kS>60D#Z!R*z7FXEOdbeO1>bOKX9?xyGAyzwa>K< zz`ylVtl>^GNXN$pjv)~l|CTMNZGq~!6R`mADu zRGwbr#V+o`$A*7jK*8QH*dX)lhDF2tXp}6+QqgVA=&)dzJ!__2Yv9SH-A>qNDC}-l)D`3R#I#vPtahcu zT;A;i?X|n2; z7IKNT=H*xReBy0*pb2x+OzCa=(5u1wu^@ZtrcJc(l<_$B(wr6wj$7f2vwFp{Lg$BU z-_?1wun9d)KRB1wL;gc{RYOioA$JA7tZ-S#NzCp~WK5qQLl;7_b(`a)l013-WJ+c{ zyn={GG>x zRF@-s-Vv#;dFlG1c2@NqOTIb3oRV+NQO1a6SvRZ?hjpn zV-4`DLCDbtVoTb3P)4MkRGm7xxjeggeK}3zeOMIB;#}~|U!q|HN87$)*N*+cLGm*7 zm;5?YMMouak%(nG2J-+y{2GIiCv>7Iy#KLAt%jUv)XEa19$BT$t5t9|!ABqXftA8d zYV`Rx<_tF$-~+0agTj@wBITI-)6f?EXQWO|^6?0~uM$~pv2Hc98->4@4Qh^b?OEK- zxwTI>nP6V9NrgVM;0c53-B#9i`);$^)MG7k){cj7C+m$+K1UbpY!*!DT$-A-NwThi z0F8BoE)f}~FrJdU;P*UXOP?Wd#j~YTMa!+tW$M*64f(J2zv?;{oejhOc=PrafP2AT zy!Syzcf_+K2$@VKU{ZSecdawW$usgjGv9~w237D#k%yJ2U{Yah{WA(%74hE8+b6mO zt$(6;Y_!9*{6k4S`Cse4Y}yP7|Kp)gVR-VTvR&qHoys zTQKU+FW8h0d#!-IxxX4@Wd?(?-e-TdY{CS4xQF>CPhfOC8v92cdBD+r51z|qBzYLa zg-09QHX~QF^VwC?FS29&bU*;L_ki5x7wHnFFq*^Ys?mz(XQf(Qa0x{i!xfaRH=P^| zL<%U;mGbZn? zTk!^rc~^_=2J5^~I0FIAuQGiq=_}majDJ2R8GA5&;`V*=pT?{RXdlTRzY2ygub~Pp z;;XS0MIy^s)CZayHYNWEXM6k~*@mSQhx*?oO^|$V4q@hNl0Uq6jn`~P>uGpX*-J{< z6YWJaGcr}sOzL@S`b&(g}PzEoICROw%j9VxMnpIxx zr)r%WtfSlWYiLE1Km7g_G`Zis`2DGO@;_|e$zV&;2lX!QtYcVM)M=M(xa%_V=H;y6soV0%bz4w2J$3sYow7wI1s=)X>C~TY z98xt=035EEKBjHXGhp(O+);YuNR2+v(Qrppzhbd>a9t2Z`xTdBNCxH+K5umvy?>DT z6q@~O#?Pb-slB$j>OShq+RNZOG2_^k&F5{Uh{8Br>#L@OkgcLr^FPnFG8t^Ppix_G zBpXLVQwL3}dwn0S-q2>5DJZ|P7xRP`hDC=~uWTkCcaW>^K(b9rLZy|uCVb7!Qhp51 zyf>}Mqw(#EZGYdJCF&(g>$&eRXTE-(QLUMsIm1dQk?oLK)AnN1X@3n*oc9-Ot5*cZ zFITQ4z!^3&QDoMs)O0EBu}6PNY4wLR>F>K$^LRian||2R{OZDJm^KqlnngweRvFn| zX^$?^N1mgD8d+>1e*P5kX__Z6=X+pGH2dy*Gosw}H3UwFmtgadl`NxCQ-w zkjn42u?0V^t+%_B?5WC7iB0clcB;y-_xS^t`~aR~UoeIYJ#jV1p5!;)c zIY}Y@4y8iFn23EEjrRyrl_YvrMCQU^nZgR);r0m8l+qkw;7KeVvIknk91OUQf7V&hm5rR+s`}5~h@;_&PRWH6e&^cys9Naw%g#_qve=dowpw8b4I8F3G$_hz{$F z48XK)dWg(swqQmEUCRcNBQNOL3;|BQ#y794R&g3k0hyZd&_rZQn6)1Nh;W0ASX^cC zuJay`O*m#$gwffzMJd(SWfms^g<~6|eR96l0><{5)C>z}9aMi<&(XUnRK`_xd+%yU z+wG49v}#P}dOy*O&{ByBl%co5)>HChMxd_pG1NT{DP(RmsO|nZv+veY4ykqfW7g7M z)qB1DjH767QME$5n$eUUYi?@`95<^4O1{ikf%cd%&?vfr*#;#13+m8X#^_xy30*D$ zUF1GwmNLyK68_J+hT)2l>&w>=0}d*&R>c&{Fu5_TCb(XJwHiDnGZEy{BiPYF5Kn{mM-s)E1__1iIvORZO% z9MKgcqKFxm$6t`hY0$`^!0iQ+fa@O4xhiO}M&-rMon}^uPY_69=oO+hLiUwk-6w+q z?O>hF2H;C>@F#o~RJ`rYC5M380bmXa?9Ihb&_6;YVIXq@TjX}@RdH%eP4x**l{87# z6ZT%SXMWE@goV8@rPbgeFm;TJIxm>Qx;;;D9GQTOj3@ADVAB9xP%5l?(A6-zXT{pI z9n|h{fWuw!H-{i_1~*AO@RcnXoCDe&3{|uP#@S;aaRy>>4r=j^8Ajzp;~d%|G&mHG zb5}&p9WXf`36=9exSV@{V&|h^cJ6`Nxg&1p9&3F!?9N@#JBQ+T?u6jE6Ncw5G}r;hb6+IS9cTH`&^#Xv z&-0NHJs$(pGaS`(NRtNt^8mP>JM4?WFYJfz85TE=SL>PAF2bFxW&}+~rb@LrRdyUr zonvfZ%X(Ebj`m^GHgLNOTW)+f4YkxRN_I?QksT>x-pNIb#h=PN{D|*_@M~eZJsD5s zGrfGXqfqWl?zP#P!u!XE$*2xHs-2fMRm0b`?Sd!)^;O zkfQT%4kLY(JylQEr(%tB%-7Sy7|(7^p%kogvM`1A4IpJp#{*lp>7Hd)gOoPxyoaB! zaHZ56pu73?AQ^}b^4ByU=u)cQb8#F9or_Z0>{z@XKrX0&u7vfz9oRTM;AqlJWrfg# zX6(x`VKiE46>I`ZPT-o-3a8a9FZ5t5U)iSI89+7SELJCA!uVaP7%ORlj6z!k9W!$x zfUOPA>^1u{j6YP5+eWBPxYXUfV{5rRicooC$6BY3>+NAeJ<81a#t?b}t`?;5OJ<5I zQ)}0AlGIl`3guWFfq6}fb|qAkeH>cUp}r(jWscp25(9d)OiBb~CMhlMSX^mVp|f3JM9^yu#06cr~L#ow}n% z<776-BkW8GoJMnpc}~e|Bk%?;9;$U&tN{@#S=%OSX*zh=MkIy{ zVe{&60Ws9#KE&D{cpSoOM@wR;b$v%OU~dtLz|no+QWJO$EjSU_Y)DxOygr<%(Y|V7 zPpUF=4NX39fTAFpN5kRAs@qt;E=>3jGkXnM`1@GHZE*eB_0>RYg7apFJ;8b6v$ZH( zp+&(7U?Lawu)Z6j{sf=scRxIBb?v1Vm316Pu-I@XR0)%^Pz&E095mc1Wb%iB(LCaT z4BGE0cC>jlx%-Wke#WYb)|l{K<0Wi)nfNPk#u1ujLSZn>F`D8BITA?~M(pNg7f$xs zAl-4-SgTq->=m?P+OSlT(Tb&Xu%4x_mwINdje1!}G84s$R1&cm9{k~EtUbAZ9vE#| zUpog#EbLJBBJH(PKpJh9aKuiA9)5^Z4WP21X=Z&mb;50Kin)n-tt+n^@Nd+HX>136f9H8`U7j`75&0PNb0LOp%tube8yitH=$zfXVs?az~Ee|q}# zhttV_{Oz|-f1JV}zj^kjXFuD2{Px+ir%!+Q%j<7$uP^_`pZ)wpR;G9Oub-d(z%Kt5 z*vPYIfBOFhcC>LW;F+3;VuDlp6+}P|YR-~-zkOYc{u-WT*a3mSAwjeI543T2JO1^e z@^|&`qn!SRyIVdL7s`OfvFyu08#BJtVO};jW2!<@9zQwD^2dl(!xHX+t&V|epK+iX z&UC{x0P$x-`Wg_WhVf^CjfO_4fr~qM>5oxr{nr?!Hb$w9QEEfl9HZ36D77(4ZH!V2 zxyC59F-q+KQEGT-Yl&409Gt=Dssq*jmEzQ-USJVBiy~2G>Q>ybfB=*YTf$Zg05P>` zTL`fwfc0U!xyKD*qh-wXF@A?NW?yx|9kZCl)s4U}Y6_&csT zQ}{TP-*P2cvOukfW_8)*!Z{S1!{HuXMs)}{$^(~-$dxFQm=x^q#vdKZ@NxXNPH!4f zLqOtc2*Wx`b8fI_9)nM@_c>&q@oSFCMA2mRwQ9mD)Ri4q&zECXo&*n$nyIy25iId z!^)0D4cr3?{9T<=f-AQYI56w=mJ5zk>7G`M=o-FMvd}VjpGlg1;Di0Dd~ZYlD8+oO z!k{|Lv(3_c|80KUf)DKMw*xC^-~(&)z>b$fbd_ZxN^enu##rEKdqTDz1*mAPf3$!| zVBYr56tvSjMUMt>(S}W7XCwslU+jS3=WX#ou)zgea;b_T6u}poi_?@!5(P=iMDaYS zsN)^Pjcv?GN!L3<9n(8SWEq5X%_;XY-=|d{Sf&7H03zsFP77L*KN4w5dbasU);A{N8SQ#|YY{h>bv&bYH3&j+=e{?`&yLCDr@UqihIa|!b zkZg{MdEDe-H#8QKd<~__E*L27uB`2(%O-^HHdJpbz$kl9(>!5vdiMvgwV6G2Qq1F% zo6EC{*O${Y-UnoIAc)qHAHl8x?(CO!niOkMk_U=)77g;e>DTrBatKqD@H8HIm=g7lY>#O4?O;Wkdwdza)=Wt7sBh3*nn&nB57_b_yu>Bvv7)e zPer{?WEmY8l6@0!(9i6%c7R_8fBl1zsUK%ODwyg?0ojCFH?P3?+;KG0Qmt5~c!ZW+ z*&lG|vMzG9z*9xcV!*yBc+4cJ_*1D^u(K?9F z^%~-Kb1n`#FCM{xvB4nWmn{ZK zegLXfe#`7DvE&vAOfMfKEal#Mw4)yM+3P`4CYKqVCw@DZj@0FurOn=h!FAP_?cd@mdgCPE*+EsybcmnG z=9-zwwrb~!;wh+_?*2$}kW_Y~KK+d?E4eo6-BvX7l2fTN>y7I$&GQa)&ubCBo7*vv7u}A_OZ38p9 z12eV5lRAR3vJaJuJs|8hEblrv+j>AHN^g#JN8WTmvj-K%cs+th#%qXR#%$*hy7twI zZ00xzv^Yc-hr-VtSR&)KR~GZI+~pxj0AUFL!P&_}ifMXtlDG4W16pj!MDEQl4rsPA z1Psb44rX*=xdr`M!vUEG2TLXh%QPLBUs`95`n8+k@CQST>4OXzNA)|c<{4Wzs#m=* z@tAc~wEu8&1^fXRBx%A8EG}b@ZJc!gqSeXwr}8 z#fyKLUOd^Tt!a&Mz&t=r6Y~&+4B)=i{yV$AB2mF0bC@O)L7_M5*%)$rTQ%CE)#<2G zEX{jxa7QQJXYa&2h(lmirZk)A9Sm+fN2+=$uqi~Q#OzfRtzh|sl(sVC|0r2Vku+LM zL>(hojUAz)mEGWu`r#UP`ZxzF+Sic1S+~%vR)q&OfN+<+ z-E;-Ho_9SylTJQoB;`dR3ggXjhv97#B2V-=krj9}DolLBj4883VEHLQXoiVKeYOG{ z7z)F;zT!Ds(R(h+0$WysJTbSPXj!=y{_@_|GF@Z-12tvKoK>qKz{ng6YnYn=&i__( z)pXf%`H!qI-pgFe+V8w6X3Z_&zR}bi?`LppSH00xIlv4~n@sXCbVtA5*k_^rMeB`~ z??2(u`s!kM%mBV;rH`!Q1@!OaYbtx#;A@)EB^nu?&aS_H{YLtZw@v7QCg&p6?6@6{tuY->&TR~e#5_KH%j)HDSewulqcKUul05zX z?A5cDuA+FHWi-#%Q}Sa*zCU}_Xpc)12o^gYbWj4W3z0IlVr84y{NsO-7t2!<^B`DN8tj}>c(^CcGKFHMnbvCP0e|DPtzs_b8Cwr)#e3~S? z&;34=*6h*#WD2mr>h8V>REsbL0`|tq3w!XNk~cz`QYPjOmtg+%sQ{JH0OaHCtJ#!% zzhasG3pGX?=2eH2ADiYq&071C$;*hbQFd=!zsOq~aKX`vW=nst3MGX|5`7dX^O&mI z=%@<5T(ZJvSm<}n^h{AowfM?#L3FUVPR%~0?|E9LBrDT7EA+r6Ps>EnjEPbojjDz_ z25?6H$qHfUmNIG!_d5zmESSuWl!*xfvnJ0Xk#;Z3`m^nH>yJZ8d8Wtp6*={gv^z8< zr%QIrRnlR*Xpr+3Q#0JNkba#g{eFA@dv>Nu*EGI z-qw+h2s?MsK*zAk`k%Hp^HFN$htgI&0wW^X=A1PfZ^;?3qQOL{-NT? zf=koxT1P(wf^S1Fgs28T3;A@IL^JrSVi{1_Y)V3Ls38&v%8}ca?wGeG0A3^8`PXJU zTj&}Aty%10DY?&D^Tl)pHE*v zL_JG$wLT`_h{}7>Y@8PTQSxW^lqYa7+POmla<|)jDH573;ryK6 zEl;9iohxyo5_$6L{BAjkL?#u@lm}nCH$i5=aoQKM93`A(>ZgeAqFJAxZ7J5+p8K=! zpxP`)R@K`W^>X(b_3}|b_BA@{<)dCc>g6Nb>Apt2{IlUXA$mEeS!bnM`GYE(+hGQp z70EHc!co_nir5s=t$n(uAm>#FFbs|}IVCS40Tx_`it+@?cGS;_ey?YM{(RmTdH1>x zSnp9;+_USt;9DpeFOx+79bp5;!{7#X69`Hmm**FMzVwZjX0cq+J9eyJ$g7SJQN*m6 zy5|)9Q1gJ|X}Rm@5=3DAt*eWVe>Ayc>sgQN?UB3I`me>@VXgkX;`Wh5?G9o#kn#76 z*H2lrK0?+|Al<@*<#0g>Vz7TTgb2w{M5#XE=ogGZLU9*?n9}!V@&XybYvVvNcUmr2UAfdfEy@c{?$Vy^+9XhQAbe7HsMTx6;c$wq}n z)J2d3nz&vN?MJ@hq(UmwonphVcuX1`{zz^-?6Va)&0uNPqGWXrSJhxGqK^p$z|w+A zfAGJN#30c%yEHWy0O7?ot{&^T^&2kxx3PZnZ@a%1f@ zWy@ahoHm|jWdq+trmDZPTEHd9F4O^trlB-oYpGG2e1k{NsiK^Bo^F+)~ zQmPx);z|{KUMf*YjFLK$d^w>-wBm|Isw~(E&G{q}*}c}HMJA_d{A-!BXhO4if<13a zn{G9OTCMMfL2PWxE^8_mLea#Dn5o_;xrimEd!zbOA+Dx8)3jO}gy6PR0yIz994X$2 z_K#Vfh_znE+l$p73*!rP(OuykCxk6+Dk=-DaKNHqEo;Kp#ZLWolWAdh+@w0p{&7B8 zFsz3u`^u+Q8BTR4kI7lW3Y9`~P=RG;X(r@NnXsa2xs4z6>8J^n7_ANixaEPlgxjy# zm3|sWv}tSTEIsViKHr8c8$1qB$oSq!!lF!J7`dEw_e*^#nY528dF85%j%i#>9nYC z)ey-8HTaNJpLfe#WO)4N;Mvjwk}t@+wy(VFK3Gb-Vv;p^{^FR-OZ@D+DXySt%_DyJ z+jN^Y-lqTKcw-1$5>bMu(gaBYxU1&cYGVO+HrwC@cIbV}+lO1-U!>tM!h@z!zp6zyWJY^+jsr018?8seY<;N6!MK9TQ__ER`ze=*6q99 z?FkU(SDkO+&iD$<@%RH)-f@2^=ami4HVj~t^;bYSo*9J{s^k!BPq1og^{noX9g%%h z&mC^b`Z0KY^Fh<5`=~h|lmGkwe)VtvVpp`Hg_0*x5yO8bb<$x`CQMG(G)?yX#_8$l z>6ich@A&`U=8KfLk zAimFkb;q;#D{@{+CDI$iNg#_Aj5fcr229OPXqE{Kq%o`}LdRypipi2?Q>`=1OP<86 zfX3~o?@y=Ce>cU{Upt&>-RZ`&NQhsVcnq>Rv{nNAO0y{4qh%q={43J_5H@LhP8C}M z3K0H#LRv@;caT1{(7P#38GEf3{iWsRD{jt>c~TZM*%||~?Ug8?K6vB%g>75%Y*{9> z*nIg{1gm;}RhN0x^2}zE=(Z+?a`(@%rD(;{D!D=AEIYft`tEnL#?KpyLUTMO#6oJ7 zO%yTDmRRO0(FgQwNnlWTYHrtotX;VJsr_{CEY^lBZMsHv?~PIsJGs_|0{VafBg5k& zm$liy&B8AhfYt#5cS_K|FD+adcry#oRK+sm|E$`sfKd~w*p`hzD=bB3K{1XiRGEV5 zuaGix+QVWatx-cuuIwhQow7SRVk-sVs6|QHIxWM3_soD^^ALz(fe*0QKt*U9cy@C+ zyR9r&ZOSCl-Eqv+Q=kb3#Ul;CN+D8clVx!(c&40iiOus!IZwHgplmS-b-$K?S>gM|62HJbti7F?H!O~BJ@0E(3-Yr^z0 zg@aDT$J$McTfA)-4>WLr`|hZ?rj^S)Ok#%TF(%1MJP@%^EHnJoMzrctc3NL(!Rm2# z;v7(=In-C=f1jTI{PgJ$r<4Eq+i#z(pZ%%&`@#7;cK%*Df2Yph(*0dN`_r@kZ&&+V z!c!gwI{8XJiO3qYJy%bg1dN!*+NN1|gpykH!?2nIU*@xRFF^u-r`o2Jj+=Zjoz(5w-%ss{0acCopNfIse3; zJ*9I|sGlEbkv;p<&YKDtNZ_VI##wD2O|<65=uFt^iHdJdtkEm=PjlAXv7*@cVrP?t z%vdw%sTu44gKhI~PwgZ=JUnb1JYd4=pH5JxJ86_yJ+Te_1Qa-Fix;z(D98_0|M&iI z)sp%0ud5#>@Pn@>`ho5U@31g`82Et5ZkNNryACX#3EKy2l($7#E(UKhpr9H81C@Zb zJsp4}TCZla+2P(SFZMof@rb(@&DL+dAT~DGA`G5xvLh)qr}*zzHfq@<+*@_YPl|>= zglB3ZiV1$+F~zoDi?(=y&OV+Q6{dht9m!)>0Om6sv_yv3Lu)X!_ssU3Y2A^jve2hu zjDg~kBqCe(d{|PDc(=a0cg|Yl4y!cWSl^H>6+?qO-Gy-*@hsydu z$g`5!DWVI-Hloc;^q?(0NO)gX%Zf@8C5(FO6p}Cv4X@8-N->P*Q(?WYFsR6IT~R<` zWhg~|5Wa~5ZAak>0mhXU&Q;@9N5S5{mq~kSU_mQ^XJ~$^7}P<-{JL>(PFvb@pE4r6QRG^_Ohf zv0RIPw);x8>|fuZt2|4uDPU+>mhRlkCaj?g2eDb~b)B;*Y^4f7Dx#82cqSQSOx|~z znSRX3k59g)l1D#2v8XH5@l6}|$&Z;Wkn)v&p3#C8KX%{(c~^~xeDt4X!DbgIEc${! z&KeIpI;IWSfCs>?z$3tfiS!O@o709I8IoDn-64dbCIwRkXZHYvi(X*RO%PKgsx~s6Ko&uiiwt#-e-8)Jk+()S3+P*RVD>7HtX*923TY z6THQ`c7*|WjWJ!TyTAcy&_`2bnsR+W5DPpnvuskz6Bzahjr1Ov#JqqI_4SF#e4QXZ zs~+Pd*rN|QYl6e+yt^?*uSwCveo+Sv%zt}pIJK=)cy(c{w3ZXrbv1mQ z->Ni?^SE;w9rngwoX$HA6!=|Z=^O>Fw?ShG@pJbSvwM_Mc7qGq1sTozir9UsI?y8& zunVGBNb$PIC|ws+xGtn@T|Z`YcgeZ{2g&eqbu7E*g~(Etg>eSGpHO}}Z|=;Pv;q|T zS^Y5VTom)@3X^6BUo8xpG^Fg8^TboH_Uf9@I4+o!$7p>3U6^N5F&cwM%mUimGhe~L z)#>$rm0fUUVX5JZg2~m@kTBGn<~uczpR^o8->yPiyJd|c4=Wxuq8}mh4n#GL8^vNk z_xl_}RjtwiamT}iU}fS0o3(MiLX4^OD!ry>!rCd_R(7voQ_ibC25Kcx;KF$g)RMF0 zfv#o6kRCItRI!W>SE2B&PeX=$;{EK%Ng^I*igrP8Z-ucZ;(dF?rF4?OnO1UEVd$gQoK68 zZ};^$uGR%T=V0c={s=LZ_WHhC#uV-W_u9-F-tnFN z;Ne3hVW?t{hAFlaqF8XGdkC7?rz-#PdIZRm0rU&S2zwmhlVIqRaCEQ`Y_L9Lu)xfR zJy5|ydCmaOyO(}#Uim=(ySLpTzIkW({*~EFevbjDg@#9+b|6>&SUJ-@Z(H)GjcK%K zIT(t|v*o5fV`KU{;e$iiyWarSMvg<7p=z=WH%5hzkBs7L6kna!D84$s^+@8&Z}`IN zam-(4a61{n@EZBe$Zz(e-^gz|zx62mrnl*sDjKm&vChQ#79$tAsNRAd0U$S8`!$dN zG2uN~^QFWB!HI}yVp~01vP?N`@9fKF8J#EW&$qYNFM>w(yPPHqDNBav|yX~SGtTRzGs7S5aPY+0!%6Qp&`2A>N18se1CxJpB$+Wg?NLbRt2zwtTW$a;XkPVgV9`bBCg?Obb zZZ;Xwtol9#MqybsPl=cxGjgRYACw!V&hB4KzxdtsblP>I4WNI_UbD@FrurII1|mRC zH5ZzQS(Q_zx%deN>F1Oekj%H>2{1tsW9`#!D=b;^dzM)*fPXz6I2KQ66)%hvR#7q- zV*6rx>PywCuOOQ-Mm!HWY1>!)ny-iK`a+)gn&z93*f7Qy7H;)OlBirqFMV$13%e8sFC;GDMLify($+w>~+Aoa~^K2Q?a^ zxRaqY4^`kvh!S5fdDP>BhUK`-19f>aK%XapIz0){>q)q7PjSrp)kZ)@_Vf#5Pg^#IpmB}7Y3L_H(#S}ca)_^a6~sPe&8rid6t&ST z>U(fBi|$9i(Jb2etw(ahSl{tAEvTWEQsJxhblbi>*yW6@UBsG$#7Y`h2`6Q#uH-vRyYisY2q77 zPZe?!K1=@EkmecPuWyU+^wFbt_{z*_st!dMLf*Y@VE3WIpqZ0?B`(2Tb^i9vo6Gat%7-}J@c48# zxr-<0z@87_vo8ygUJfir@WgQmTVFLawh@Hix8CM!x1IsHrv<0;gh}#L>vKO{-@N_n z{~UvEH?#ksSxnx%#eaE{9fr7NA9Kch`v@rJ(Tg^`r}((miCkR~6JBZDj1bmbh9Tcdd?zhw?;%2O~ew*J47p*Eg7LnQkAH0 zk0R)7dRTP9r(7KZ>b`~rHNZND%Dk)^_Y=JayH={dh?Q>6dvmt)m|z!%Db`jw!^j7d z%LN!FTKQZFsUQwf6y)mK1gH49Da|VR=Ef&HW|;}&vJVTuIdf14@ilO!m@RB7$l<<) z`f|Q~d$k}Ud&x|*Pg{Smj-!|1GENtxPY-myioDx+2VgYAicyMy~S{0*uH>!6d{~XmjGOBmm z*QnlkhdQ#S&1+Qe-0L&rO+laFnPgE}u-P5YzXQ8mu(Ef-C1eXDqiQI9u)lkCx-i<` zvGu^<=xzR5!`=ZeTks@Ir3>J}WBBtJ{=6Uk#_;FPZ#`1@GrAq#45>r&t#L-7?*hnO z`ELhkY=@{{2GOB6Y%!!{#o75Qa%-#S(;1ZdyZPdoG{@d1A(Q6#z(GcII zwaprZ>A(fI&C(EhlP<8se-~aSW?CsqBu{hnwElzjH!fH)Ls#|Z`P_RIeYN&_% z=~H8kHs5H+D2dSx`3rMHj!_csHAYEvb#{!B7}?VpB>~rG#+$|(ZO%1X*gjo0H(J$7>Jd=uMk-vV< zr(B(EcoCiEwxfBTKouK%guy%*SSt3Yw3jz>thh5e8%JyYFUOicIvd?}hm1!u6T)rm^DAsNT7KANl7mh<}dioqLVyT~}vE^=@QOqk0F|XU3bxiaTS)9lXY1 zHvcsSv;DGy**@orJG)lK$YgLqj5jlH9gH`#?IIB^9X(L_i@Gyd^+FmCKKPF)6gn`S za?=xAX^V&@v|uD>1s5@oAR^!&B`Y}2t{smvT<@f6CT;1#3aOizGZZ|LVzB|2p)n>@ zo-h@i6ikZbeyXX{-sXYJUSx8wY8ODwuqqdg71B&U1+o}AKOVSA%CjqIqz7N9J2-K8 z7#wE3qH~j@MAPG3!>4Ql`0VDp&T(BWGf`kXr+$tUED}ZBCA0eD==N}~+jllYEH~lL zy63157L8nCt*^Eqjcyh7Ujcl~SD~O8^8tW!dhwW@WR{cAOlu*EbV}Z4aC3I^UH574 z?3kRLot+(%*MHRt!JC7k<=OY` z)1kXc|UzuMeG9*Hr~7Zi3_+m8i=0rJd5XTbas)&s24-sy6lU!YNOM-@ZC~ z-P)*Ieh7L3mWdU;$GABbz4+<^=rdd!uyerCOMKXt z`~&vq1Yp+~if-IyC&fTs&DUlDMFm%^;B-pP?kP{e{%BgWx9oQpqRhG{V#NwpZw0;h zr}RNiX-fYovS}pJle6z=j4Rda7$LAhh0+U)QK({N zGdW^gE3#J3y6IgeJs`3l_-tKl#I<+!oGN3#at>HfCi`D13cg^=(5kApMpeXDcK)D9 z*xAjSFirLQ8MzS&BWE{njtR{O)yy!Xir+Kz-s`39sINWcwuFp@LU12=~U|u$evAl27&@(^LIaOPk$Uda5XyW3p+n8v3Sv!4np08+++K+hQDp z0D%u@9doBaWG4ho&7uJPjD_X3SgOqAM9k~D5iVNC6P)LES|ywl2!CmD#}c; z81<3PR_oO@tzto?=G4v&WZ$y0SjLdklF^;s3tXReUpX?`@{eK=5N1w z`r~hY_}`Pi{pQ)9ew;r2yA;{a>ooa!P1EG(|1*30=I8%;{p#ocIeY!8Q!o8M?!`~^ z{E!H9e27G*Sf;Gr13T~YJ4N^%GBBvaiRWb$EOfp(zx2tRZ_Y12pvZaR^u}FNZN8Xq z&aZXvT4!TtUkph*Hpx!Dn8S*>W#pG~kWa6N+d5~st*TA+jS$O(k@G~9vFjIVmycHA zLV;gdIOZ5~^v&D1-@Ll~>FoOY)z$gg?bX{iKb_xPUR=Jpy*hi%C&ex@>*lOL%$6TPnV&r>ZX|Jdr7 z576OMcXg&VU%;cf0?2HgDf%9T7MHd~E=^bFo`a+@9GX`!(xR#vMeW0(^Q$EUYS+QEXt0E9qnck`bgOrLMc*E?vdL&54x+H4 z%NgQih%goe;Z$@-N6V-Q@DD4r+XoKrUhqplupXro;VVgD`R*Tv0-ZoQ|wd^k0IB{EUG((I{E zOaF8IB#Bl+NV5--lO9Qu;5}FnNy4Sp!dp5DDym71!B@7a>@#OUk8gR(q@rma)a&|b zP?-vm$jTV&Fol9?*4>)%X`?+&D>iYPXO#gkriu<6h*Ejz5ece(dCo{s&FI4&lHBe% zy{R>>@D4q$T~4?3uK{X!@Um4SD4}ntW0u*NW#*ybm}Lgn2uk>oKndr%&sx%iSsT8# zkHZEp(pE8K#f{h9(kh19ct?xuMlTW1mRoVvrXkt2sYAGYPNjSh#Sm0D`-&eGuE)ye zQpYyfCd5YI58s2M6x)w}qZI4>)+0%=QnF%L=+ry=%E9=JZE)l_<{J6U$Zy(4^x5;9 z-lj*G<&O)qJi@e$FfAiY%Lvml!n6#b*9g<%Tq8`&G1@X^Oy>qWoz3b}isNRk2X;kmv`pkIKh!8U(#NahXA^EQ{3h9>= zbhuNDFiaP%`L=D_wr$(CZQHhO+qP}nw%tAXzcX|0=BAQed(}#%QhAcSLY_T{hT#=K zIcfWPJVF@6&#jz1!P{Y=N()uk^u_pH&kb}Ee~lL#&7H_@)9F0N)O{7gxy>GE6pQPw zH(97iNyXR>DAa`0wHbiHD-w*wqi+&Dwk~Uv&KWYHvvZwh&Yi^hq%Z zr3cQU%eu2HZFZ08w0b7eIRGl&`~C`pOm8ecUu2tJQ8+rvMkEwWRWi~_Z;f0*V+a*YFOAO@o0KUqYjl}Jj4C;GK4V1gS7+rccLTGj z%*RA($}hz{f%DnEPu%`^c|bHiY0)0I%6FiSFDcH7=$87^S_3H2u5g;=|F{kihDy7n z@<7O+Gs1^d!kF+;l%b8E3zvTjspAjD}KKhjENz8~D~_S_bm z>-BbezEWHWtItwEnmp%3&^k2X6C6v_y|E3gn(Zbxvrth4B?XbrfW9)&eCofoikjpp zU)m5HgzHklDhpA-vjZg*P<5w8123}lHN?6q+{bs!RqVYUmW40 zlX^9QeqZ_nyy>L-Vq=LrI@!Y=c~42AJs^zs@6yxwN4tY>tfr;zVO@;nvewem_I2|x z>>#DMKtQbZ_xn^}A6V)F*UPFsTm*r-c`$8!;9^C$e$Lr2ZMgbg#AOMwNZe(hF4;I> zs(d8||G)I7X&*HezbV}H7o-Nbz{6JZA^;ccplolq`|+^>KX6L?7zjF5?Cw6ql!s-= z3i09Gq&^v0p64 z{~H!jG{KkH*Q)Wtqn|2+3`NAP#WZ%rX6&%l$YHCY{YHJ;6})HkuyvLNwao`xZhatX zE0P#G>_wC~1=!&cUZb=fwGT^n)D|UQ^7w)fermE=OTd%puy&Lzo^{#7X*Gi(;;oB` z^5M7Te09=su7|R#>+X$~+@iUo-aSe3t!jlT zOKIi7b8e$Jqw_N?6WA|t;S`*<0+Vh+q=qOIp)bN5)=P}HF0 zkFb(z4twzLyNmhO7zAab*z zk?lQQ?;*MEeizhb6Vh{Sq`LrPqk2Jxs3%G*4IBBHAhFbA5K~3T;}mpo`zST}c;tB` zZAl57b8=lsOX6^+%2{abMR4GJSH~iue+!mIf2SSTq9BN5Ar{!e>S00r8AoPPXbBv! z&yuhudQ3wsk3B)L6cTqw8RY8AvZv2AuWe+P;R0Wv2^Ly8kwYtgU7K;E)-FWF-f!_e zwPqK$jw$mo9QnFnzg3VUd0~F8m63h|nNip{bS#*(dj4+N>j#z+r+cJCKi-EozQLZOY>)tb!db;GyC zY_v7?tdMA8Hfih?qd>x$jgYk;#=qtQJMZq78u{ODCUC#16*m%+2Gy##F~VA<;7VO3 zqMxU67LZar_m$kgZSAP1VU8y!!sk*Urs|Wp8ETsGz-}R`>O>L~D7x_b*D!7}!hyu~ z?H#%gpAWnDNM}S7e$#upGBYLTFYWv95kEBcrs6)b;_tWG;=f1XB->Ni9!93MJnrQ< z&}X@CU>%TOUomkuRiVEJuRT1afv>;!?YA+K)TtCOx_6w^DYe|bq1@Y{^m8!XOE6sz z+<%a}0RJHF$o)gzKK(=8V42$VzAS+A$hR@heGbF+kv!3b?QHK<-4A>O<;wCzJ&LkqmfMN8TAJZ^%qm2h3_u)r;WD#1FcLXc(pS^v@EjmulB zW2T(1+U{=YemCk(ubLHVt!IL2Gs3n?ql$!r5Z*Zo;P>$jLP=*Rw?r!)=OD6Qtg2fr z<%_is?rQQIbhjLcKZyTTU4m>ynf^5xIl5we)(9+0U?&n@(w=KI8G)LsenSQ;1UjjN z`x;FO-V?H=B;I2#N!!KlFmY^ft8>?0Pquhz!ojh&NaV5*UFO1%E2QX)pcAilhQh#k zQ^YIGt@U9hu;c4pl}nhNFo;bVZv_gQb{=K4xrr+k0Z!|A3dwI^8xTlErAI|cS8-gD zREl)(te@OkB5dqZeEl=;b5^5If|wzHVTr*t9nvn>Z}YOycO3S|nHi)>DNWJ(&r6v(1)qdX(QyCtSbAPORn)v&AMf)1sIJUF%`%uz5ANd!= z&u6}3zDeifeoxxvFGi4OFtG!iAOe~-g$qk5K&3W3M2$Y^qI+WK><`qad{-Jnfp63? zcOxrm0f=|3WalWI2pEU)Y4uPi{T{XSHrGe zJDa58^2efQR9u2+8lfulszq0u{T-GPHPwhUNFODQ@cou)vnR5yY-QlaLQOo|XnEa3 z4KHo1QQBo8rHEk*L#PlRUCp3W!tynb!2KMilYVFe0(_N9_OFb}M|?6+18!W_+78K6 zGB@QU{?kdD^-95|_sd(Yr?!cTmu=AXqwf}$fVT7Rc~cE+{=JMpkfY9Dbu%U~Orxi( zMN(1x;kTvw-rYE|_5)Hdbd5PA)5l(%t$W^)3}F~~zsj#9&GzUjrVc;g26wG^l9I4X z?f5EFr0>t&DuJ7MI$P*HE9b%Yh1`@RlWA9oYIFqKo{lfl%A;P3W0Hikx_&t9SaP=u zi^HTtKu@J^2|y?4!%I(7Mv-yRGLSSJE*Me1-3A%2Hsm zsjsQM_;rgml-*L7R+DbUu{n8zw(_QL824F&L~HEahLnuY6>5&q>E z&C4FQ0jldbcS>$Q;EuH;^`UXCj#FT2o@ijh%X}8SA33mnp9S4XDF%;eEV^LC$ov7l zD{OTBaMNvU{;)BoU}U!-_PSzcbpFuuYutS)ox@c5?$QVEd*My*?6862Phx6z{qnZEJD*SAV;|Ju&w7_j!*VvOv# z#=OeL535`U6kZmoA5>1lOV0yW5WHYZ&CXB-P!ogeOp`wd9=vr2{!#HCfXJ5zKsDfp zBP71!HuP%(FM>t$1^r#}*qP-vBqb%k=Yqnns5Ou$Y$ zn2;AiF@3(e`bA8Hp`;1(xrIvK&3~N-CgPqF7CPnOv@Yg$4A6e`0Gwnv8>H-duSfW` zMKD_jGXI-YY)xMU`ynMGWdNr|)-w=6V((&?G3n{U8J217;OG!jG)Of}ShWJ>oUIPQ zLvt2r(KION{5x*C{pCbymlpM3MIIuz+v`=S`nEx7{C>^zepTi3vl6ukR!rM`F5g&e z6`4-AvYlpOJI%y$nqk33cbZ|zMR%Hs={OzLb~38%U|7@cYccyzVAag9rhPRRM00;S zij1i$K-2vzGo$n@11!L*oF z&e)|h%>`*x(OMKb3JlHnse? zmGe6K2Ew(37r3$Q7!#IrO)bP=wWyR>Ebn~Nx#s}xmxMmOFRZ=UgQ}Skl(jh;>|$U% zfHD%venj6&OyHA6sIls@z>=PF%9|#R>M9`(865ini(33q5ijOQ5fCCjibwkDkOy*M8CG!YYf&8hKaO1QNdDX zZ#qKxaq7Y`qMA1t$SvY0*k=JJMZqTvo2pWAzU;vej#!ei~aov4nP?xv2`X$i{#G99{sIzUH01cW8?R?C6D58TAR{F{5fM5{JPu0y zdjE@0NOjmLBYbJEmw#uTQ%FZ@nh}p^@>)E5UHQ`KnLHk)+a{A)WO^CB*e1{A6w^FB z8Ke^dAK2v?ufQx*@>MY4&>I#a?g7#bb`~OgM^^zAA&TzFqvk<0Jz{9obGNFkeAJf}4jZlI2gqr_)JJ9Alh`9+j=qGfrx_gYc=-jLSLp z4lTz~;U%YXcJ`4p=JRlVWQ<=+8q{nxFe)8eQM#^+Zw=nVH`2!F5wppZCKD-u#9TANCStMg9BG1_lE21kapA$eGjJ z-kdbm-&aDjoKyhm{g1Q2N@F@oB6!qE4ym__lYv}&Ov=^w&ZqU~irG?=(j)A>ug;t_ zVnXibjL3#zroBM>vYy9Q%M)*gG-;l4;%!B$=W!040OqcrLo)D*V5?iuyAyT* zQClBp350|4gEl5Wn>XN$lOhv3tiim)`^f4>LRxO&^ zn_)T6k>i5fT#rhG0;@1_MJ__3e*{|*DKLPQXy^t6PEgT_3!e$}6 zL}e5jN!QAYBfwF{5-1YN1mhI57r>pf(1D&QzLeJ)DPcH^PEtbwJJ%WjJq0kXdxldo z;gTE+x1-K4ho7JwazlpAW_YJPpxx!E{Br0fPKE_|u*-^c9;da&G=+`nHO;Qq_aovD z6rP~g5_;27+#`|2@rI`ux(U}9tDciYbpE*(91BQTZNM(Eo4)Q+7tYx=Wh}DC=q%GRg(~n z1aMkfP#El~9f15lFO9Mi6)#{0^?80PI2Ct2%S^eW+H$|W#L(!)w|Vp4g|8m%D*>&r z1s70-odVV6sy`b7&bE3_$>UmGGTm-jy;6B+jz_b4=d4Hk+9yxsV}dEt9TWntBN(YI z2vabFl%N2LVKkUamb5}UOoIjRX&?(8t{pjV^6(u4@83{gMiW_Ey}z?IDB&i++a5o4 ztOpWzCB2n?)Hx+M-5w;#ot-~}4CEh1bRs<}4F+9`dIc{l2nG?hX6lQ=Znv+%4gt0u zxT9u21NjOo)|Iz#;#3<+9uq6dMzTbYSSBwb8=1e;!)s@bwvX!998{m_-j71f@R_Wn zdEY)HdBI_Z9 z9#)UAb~{vUE!O5+g!PXmAi;PPU-Kka-rhFIu0{1;>)$bhwQpK+#Mgop!3(T?4t3Wt zd&v-Gi~^W&ByV4o66jfj>NR*>Qrh%@UCnU8B+ z)beRg0*KY(pUBMPtqP;7pSwIO3O3QdoftMG9Y48g0B=+iGV)1gz1a!RrN)6h!Waak zm|mjU4Is}ai-(0NrcUyhu|@r~Ob{dvkm>asqbO>2WT-i=)mn5j1y>kECV+^i3dK|U z6Nn=yVHbdSiOd^a#1WZw=n#+Si$s!+!{RaV!}k(VVi3q=(s{3U$&-9hQ+R$_UPq`T z=>bV|hJA`qYyfQIK24Dr$+b54n!(QF6F04Z|yDM6xd zYtiebm`qlQuf}7nHu!n-*91rGDP^9f&wnzEoMaQE6XO?UpLrjMQio|%yCg>%qjz=U zk|Cx%30-wBswY*|dH1I79(+Vi?Lb0H#hstuJ6y!rFJRW<{^u~_`s!OTZ7ptgF!J)% zH375k%bZF!S*A^Roy`SU$@kPn6busLmKP%?_)vz&@Knf#$AGp5Xf(PNP;MJ`ZC28f z5tYqHq<4b(>maNNB9(skWKl_A7 zO^g&?bUJL=zBt!y;xkHm(0abQ>CIGnyFala9Njyfc2lo9C!=>c918rKd|9fq1UTB9 z{8`C)jUWlSR+I7C6hn)XUd>626Z7-OkGPpL%};#>kseEj)XKlkPqq++iL+f{cmyxq z7GHAwcAK#l=RnGBOPe=|(YwrglAcR~5f?j3zt7LN6mqF9Z{cJ!onFI9cRM_X67RRT z4`g+2avw_TU1fj~*E>c5&aW_*2tm*#t#7VkMUx~W-DLxW(j~()dUT@muKRsTw z$-sL&Xp$j!xlkm-ZgU}vhgql=<1oeJY&K)U9{{jM!Rz_X9{`aD#-F7l#mj6o+3I`D zcAh>2hWC!9Q(}Ac?cmV)JV68OJJOCejAZ|4L}p_G6Y>1&-FFCwt!+mWT{PHgZT+o z=>L}ISuVro1+EBdUGUW6EHHp4lHU9}2T$@{|Bj)**tO$PR1b|Z=o<$7vauSXoXvkX z*#q2I@oF{^{#UM8`zMNw@C#$J4;h@2fdT#g9#)m6%Zti#5dTHWYy5q`+HJozPJBd7 zcwg42Ti_>sLQg$9#gz;mWpPY7f(~4u#~1D%eR)vhadLUG%K2yiqFA??O>RU0P0j2- z*@AsP6WaD_;8nhh0THB;GeLUquf`}hiSeR&!_CF=hI?fo#@U_*x=!fxRo);p{qMSf z;w0z*&@)OoTmOtui6a&UueDo@CWdTOv#tY3=W@$XgKZv2?1g9p^92d!AKFf?W@p=2 zS6I+-H=Tl?SGG&Zu*vO$UD~T>|IX$yS$@m&lfg(%kLm#zp#rVApyD86VDo06?aa<+ zBGihVDomWry&D$XnS~hR&=HO{nCiRYABt-l-miDQ3lq?jXBz{AP@eDWv7H_^_LWS- zJfYaVVZ(N-@!Om#wEmLTuMaxs{l2R30U*rpT+=u#n!4CVYD7{t-J=<3XLnSoK)bwm z2#Sn->WdUgc4;Foeb$4G>}`T_dWm}R`Iq7gg^J1?z;e!hAT2o+LD6}hw=1~}g*_$J z8=(l^p+2C`! z5AP1SpaLmOKl<;`n&@=PZVxn?Kwq}Pngb!A=hbmrYIWS6sdYEx zY3GhqpX>g&A+8SCEwI69dmiOLr1)lqH~L;1p#euE$DXv}RHwz6#`~ za|3p>hyP%GD9$oMV+qjQ#)|fuvpzg|rCkZsT5XAz_g+1iqWOCdg%T1isVSqSNNQ_uvV_r5G-om!U7f;^B)atHW}BAL z{sE2%l;`J5wQ9>e;Ofd~dqRO(DK3V%`@O9Dh-M9Np&A+H!d?JtVV{`0Nlnc89l}q~ zis*BjTRD-JWgEhejvXF`U8InokshJuvTmhRbt>%Y`!#kmTN-?ehYs^*^IjoC`ksi5 z*iMi-~N zT;<@=gR5wI45l?T@IC`=o-C!33cZCl1{JFfk-e?baae~k3r@tyT3L4Ya#IL^P>mNZ zL5zdoR%s)hSz!cCGR$C{KW8qel9Jh*R?BoNuL>G`zBP^P6Gmc)o$#T}B)}=ARq^yS z(kASbErD3u{+rh4s;XUAC-+BjU#rJgf0pOH8nkU$ljn86DhX)8fWVZfGV#!m%w6!0 zvidNOlhexz=c!h{OGIOPiJFQkeZ5lRbExf>OB+Ai#6;W(>?9B4GZadAS3n-NT6x8Z zQfJ{~i{p%RTgcByU9Um}TJ!qfyl14T`nDhgysDmhcBu~YPXF)O92gU|FRwk2GWMuh z<7!efZdU~DH0oVd*;V9J(6c`yah0}>Kh>O-REi_vMgil$#l#Tm^G+l~voPPr7Q~t- zQv!uW3YTf|&D;dH+eFJ;u2}ytza4}`KDH5UTWBI^n-eKQQdH371r!xp=88~WhTus; zh9=qs1t<|<*CWhZ{^8RVP-lmX~kCDelL^ncmrG8IbIS?m%>^k}DHc|4QlE#E8*Z(MPFL2Fh=VL!UEf@A$wOOFE5 zgK?Qr*Iua5rnf{Hp^e;oEa6SH0{Bfwp?<;m@)+{kj;^ER3`hFZs(Nh)Ln&-=-1zpCY>s8rGrj{TE5Rb@3(o;wR z%yF{GJ>Ra-z#`?0@Ur=F7IC&4z;9P~j~+GKgx|}(jjo7~p@_i)$+*qUVpvlhD9Szw z1UJ+FX3nxMG$$dIYLzn)(09RlE?~tud6>VNDJPa32bNyQYnTB<0G2>-F zQ7fsZUNTr;rdVye&oX?_yn2bS(2}iALFl=q(zbSGK!EGOqE90E)^m4Bj5<+ySX6<}X=b3)08b`@tvd~LgApH`GPqig=w~my|-ajDEJkS2{W^xUXe)N)!D36hIh$J=$ zueSbP*s6!RWVM${5HD2JkEEmEYOa}t8R1EJgg?BcG;{O6NYaKq^J)y<_hYSLWiiQ$ z?-J=220*yjDPT@wgfu28z(e@_`*5V&fUV(f{MTdb&7QvXI>M{oT%hc@uv={p zb;b^Rc4lsvg?DDG=2q`cc}(5Ber_)-S;qNZL%OpbWF&1)l57}jblPPM02@m4o_j%b z51LUiC{fVd1+MLW?;l!?BSI3(BE%2=NG&5U(hA4JIdMir=v)K)nKe}Evo~!0c z&B|6AwqB-*5iTwwnJU{&EtZ0z21|Y?w`y|UCX5+BiUHTTMcB)Za|oe_PoHn={J%B$ zK8|A>pD8?5#YfcChL2A?YB4a*(ev7a3qRY1An2Uv=*#v{m*&FLsll@>8Kglvb&TsiO6UxfJ2(Dloc2*^+&`LI$ z1#psJ{1}p)fO6UwbNu%%r#{|p`nroz5tGeA`Zb!aLT_j~Oxv%UfJzt$RuW>Mrbn41N}KkuW>Mj})8)*Bl(*IKw4cf?l4;XlicuMw+Gl$ckL zxLV}4`Z7g68Xmf6D7GGEB!NqX6_Xet6gww!=URWQd%ae^Pg;|od^O;+ZN2wY)KMHY zON%=$Zvkc5a_{Vxq}Z9Di2Xi4&;R&V=nDQm_!bUOB-o|QMHh#T^sa|$2rPt7xYk|QGS*g40mhq@OvCp%49?Qf_JPm~kSTjf{U-Gz9pDhoU1UX9mS+`N1xEW;EA%_Xbgr0t|aj-BX-17O)e;-L`~ znd%9pSl0BA3iJa|qkpT4bqbB7XZKKL^kt#9u4Bnq>oR#0?7_&@V{)4socYzzlsPU^ z?pG@Hm2G4N;1@Is+T;I!QaowHN|n!wEvv_Rp}U<G0k|poitn{`F1_f%WI-doc(D@a!ifGR+CfO zJT^G_mJ+JsBEoyxG*KAm0V_E@tTs*2bFymceSKHRmkP4e&EM*OQ$2t`-XN6P_ zK1=s|?&~MyZ6YRC#qg+6gX}n+R+Y`BjghwLo7%3K67|c0X8Eu{vdB6{@AHYJ7G#lj zxd$RUoCR;rLs6EhtRemJ%_(Z*{li_=>{x}-EoGRgrMX5H`&meuj@`2YF7efiRHT}u z1e7G|nFdKV{@f;{h5g|jH4e`3AofN@P^ni#7P9rksRdmQ{m{nD3Oy^RmvC9UR->sH zIFg7bq%Dp7R1%FJ4Yw~pjZO)U18dA7Fhi39sdx;9eFP5!(2QfuO|b>AnZn>d0#z3y&~o$2(1+Bs3g@MZk)G$@XJ&}NvKM;(OtobugBf+GA|CJ32B{Onn{lO* zrA-t^Nzr6(PMHZuJEEsfpwgd>rMKatV@~ZFZtNIXNbU@Ek8K!2uCq$ww{6500P)+w zOclVTIl20OrpDYPXFWI@m(xx_WkiU&)M4^>(8-h#!oDZnV~Dt~Q>ptYXZs$Vw0zzT z`rY3zQEtQ(-z9jQ|HSws)kzI`jW3}x_|9#m`k<8OB3rg4tIL}Ws86^PGM)M+C)Xf4 zTx$S3^YS$c;8jFTsX5Ys4VvKZM_5iq`DQU5)QI#fUWOl2G}EGktH2d4;)PDKy~H)| z+J2dVA^$Dv?tw=v2U?ZzxyhXyRvbeVDlxJc;*kA(fGgBcQ3)cOW0}3Lc zy~ekA-br|DZ+TTX4R`}=3X?1yAtm3d=#uwPdJZ?mpt1pK+3X<`1yamen{gF^YFCM{ z`_D@W=;NkKaNBQ&{>H<2`foH=%=S1#NVdZZpJoHhw36NUB_GEgJ9K2@CQ5axce=Pv zSY>gQ?QLN_zp#!#GyxVBYwtlshN}!i<2NNxlVXgDsj=ElO4D%)iPcL~a3W07ec0D+ANUsBavX!}*)`3u@*l`|K0hPaiIDi*T|3*E8DNPh2+QoN{>w*^ScE!? z^tWQ!iqFy4H2y?hYDETuE2PRKcU-Ug*ITs#YVtr&1NyaMT{B3e`)}Rt!9hTMq z5Y_&=I09N|uQDlo`uXLRk+LrpXa0T!+-N4VhUW6~!BhyfFa-8WAP;6^l83#871ie7 z1o^UNJ*QAt3%xate1~0%o|0`_8V(L)9s|O)8xJc6*YS8LkIq(Fql}Art!=b z6k%X~4*;>0yB>9L-Aa*rg6Sq~7)@)i*oxs2UuBrCG)mNN>msO` z2o*1_!w^jRsdaNmzqP=8(nL~I)4Xks!35tQwWX<%W!`8pWtDZ}?%1^SSM?Ud6%HOs zvl?B}Nl%GW!9ja@S5Rj>|BGo4AnI;CA*8_yJ=+org-=Lm4yJWnSm~2vi3)h=uENr| zt{3^%!@4dRu#}-yQAh4m$}NS-WP{0$&7tX<&}MtUQs8<%0gw(St5F(O#rPz0J@P#s z4}_%kH!pW1sWb}v6l-{OMGVX;4qz~f%IMpMU*e(i)2juJBFoF9?X1Nej*LZQ!BFjs zr#rmsVpf&B$^OBJc#Tvr)K&Awt#NW2=zw=h!1|WK%qUrV<%q_GSs=5W`z%xM`tllp zJ5YO&_BeCm$n@nsLvPGlO$g2pW^Ig{rythcb1v)d#Tb`Q)!tE#Z}(E1+h^?Soo^Z5 z4`vapa;pN^W3vKSB=&qc&<5-O`|`bHe9$=OJ?|`rok+Ro6RizQ`WbN0R`33R#!k7n z$iPWY%Sb>*(Mu4(Brat^BCw}*5E^zd!&F8dd;))m(b@dg0j5a$By1y2j=8uKN4V7Q z=t4_8V@A+3=t4+nLQTWGF%YixP`fM$I9miPEU|?cyMN&Au$v96cMOH1c4}P}_{{B_ zxTek(E-FdSjU1$P$MkH5=bW-iiGq=iO(-0hN?C?=9E5C8y&Z8e_7<)#@k^DrI#%@W zuH91dxoK4&>kqW{Q8#rr@aXkZCtrIY2w1~w!pd!3j5MO!9*Xe>%9;F5KJxqwVfd7) z#v2hISMf2&nOv6iWzklc3Zx+gpdmWi*5+ZtvkQ@SZi8pD%d-p2JRRh1Tl_z~oHEb# zSj4eSIxrmn%lNwqnMZZLqm&qaoOKZ3vYFkHeki}2`akUw7ePbd|X;m zcfIf@z{r61Y`He_=4n~Uo4aw zONEwl$E4m&J)ijrpUoQsf-iC^7K$RqO60|W(nt1-FBg)-RM|Fcxirj*@b-3YBQeF! zyc#iXPkdM0cP!@Xxt>c{e_4h^6<0HaLg@LVZ)Mii(*s9dl z5$7?^+(d3&r7||-8-wc%-ZnT0)Ize(W z3yAQ2y|rZ2vg7_udw2g61!}RZ0g-T>cEujaMWL|T?~GvXC@w5}0%sD5lZqNg9@q_x zQktxi>|&lRSh*F~(CqJ+h-TT`cClG$ARjc1(b3R<;!R(<*)f|Pfqn$L3<;zi!(=nn zrr8pcYHA(LO)q?seq_(xDojb1V1Lo*(L-D%##xgi_oS8?71X4h*XSK)(JHT6M-@q= z=LRaWZR$g-Hk}l33REEg*VIQ5w>0{~sB4Bnz8YFi?xi$ph|Zi0d%R=~Nzmcr>@wA6 z;c!Ks&}?aEkHryLX(A^Y+l3n2<RQa0stdv33RvbjL#Pc^hDuxV> zP+2ujZ3P;mtkk$Z&_kFe=Cps06w*-ONLP&j1{>gs``PDVw!AOz+mAl5;aq9ikU6{;rXREQ+Aob@Rq3EW2e#FS{@mUBz z_zc)CRUi4h)G$?rE=N}Y8Wc1)2w@^u$TTP zsk!kinB8%yXRQRvxlV*;#R}$NCB`X2x1YSiS^ihKh8&qU!tf5j#iPoR1)bhxZqOh^ z+V$0A)@W7m#@8t9y0HJskd_^^XcU_!MlbW4t76` zno;PX8e*7nYceWG-{#=mK&JNwk2PoU^S;g)9tjSCy*=7+nhhcy6Se*o|pDwEoEoatA$h+k?9y)0?wB>=36pUIqYyb)oOWIvQyHU zEnEquJ6J-AP(b=ECUg2zmS!0o_%BJh5_}R*vb_Y|JvWA>1TJ#RGdjIyEMjewMD?nt zRA4(z%G#fcdKb0cD>C2rZ8e?9TrAI^E{Kv6&3py-INS5DB9I%l0*>7b{(Rhq&?w!&DA~uR1p9M*S+tn+!-7u|H1($Cd0un_nV&a2-@}e`t}&3T{rG< z+XHxPwur-}5QlSA0m)_}*7;QU3&d4z7xen_EpfmCozcf07=eF)$oe4;P|OdR{1YFY zbK5?U7)%6OWMbglykPShG-7mrmW$y;VA3bui7uC0yamkeb|f*3ZdgbC>3uv8-{)V( z9nYm1x?{I+2-w9f6bo(zI1GH*!(~5^{9zE07cNQV83U0guPbUN$_SI=LY#k;tH=?s zg{#O_Frh=BaGQrsdKzei>!T(f9gN~P&i&@V0(%@%1QR*rn5RXcuo00xr3TKzT>E*J zsjPui$>wZ^zJW3aWi^*l=&Hb^UT-@LP-8Sb&*-T2azN^V1o((mb3rPC7%8Eu5fvL`JTJvVd9Hb&GwaOD1T&89j;nP17MB)btwHcn{ds3dzYD=FTUxzS?-!^e8M zx3x4c>e}eB5s!^DFDoftl#iJ-FX~4G!-sl)%*2@W*PARatLorKyChX~ioGwJ3(Ehj zL_#WKQtWv}U3pXoKX@Fj8xUv5+qP_~gPBwZF{lorXUi8bOg8_6%r8Rh|0)*FqClJf z5Tpqv_CNkaTHrM@S^nP>Dfa*8iVcXdXBm42B-x_Xa&{cpd-zs?j{F?ha3W_>5Gg|! zH=SXxsxf&0EDqepSSya8E5i@)HI*mOl@Uid8fr9V<}%oD3#RVV`I+pvl_o9O4WzT` zNM_X$NajW|*$e|Zclo2hZbpG!3u)+cqr>13T&awS6%x z4!qleutqFlakZI4{}~)v7|CEWuMT5$oc~udtqxag{-4g~Ib%2xM<=vs+Q2y@`u{#n z|DI@?!2ZL6N&U)yo9A{$GT9BJvOO^@4x^duMl#uzKH4k}+SL78&~J|(fP{v+-%verV$uvYVu;BoXVI&kI|&-h`lQJ>skeQ zURHmuTLReBuk2l8SR0@5B)u=PbY4%kVZST^ci(k^;WKcAP}3JM$%zWW^w|XNU_bUR zh@{KYP@R>K*XL372Z%U1Jl&wP+OOzSv%^U=fBRW)grl)&_A)v2+3{c4<^RM0RpXWOgIyy%%7`yluB z2hmL-#T7LjO*;XyGtFN1`xuGjXv72*1`=T|zVpE$um=WWc2!B_Fe9JJ%yQ~zVBB?m zi`kr5LJh-KPUC0VSNbpDd9%=t`7kqQ%mwU1X@Pa&nu&si$z&m`aSa$#iWS1p)oN`K9lr{0jy2h=L3&;oAzv98YR=N3HlZ!~o^Uqf|BGKt+ zxmjDj!`s?J2kYiJFq|c2=bP1$s!z$mBd)JxRV5yu%tzcrIHpg&EX7;DhbxGo+o%W4 zHS|MJgvA_e0JSE7rJL3nP(N6rh$csUm%(M=Jq8+Sy+Lc~rtYaJBuo6VwR5F%v!!PC zt_6{)rSfQAZ85fTGJ2iN;j$HMxla$-f3S;X9W#3^{X{fcsidFupgRPv~d-et> zNwE|gywhZ-KWZ?wN?p4_hGH4Yz-3Cox=t${Jyx=>s*jq&XE}NKEH-6V#UAR?1dUqz zq+47Ldv35+UBrate7sb7%x#7deb11Sn!qV-$aEr%32cKxOM)b&t<&vL{yJ=Qo> z5T+-D0C{uoK4@nwE|?5L?ebEC$T9##K)b)u<>MfJZ4A5ppDqZ6Yz2qrjlmk`>Ob_Y{!qO`KW6YR1P?KhiGcB{hRnVVUu zS+tZ1x{LyiFoz+5P^lFhlTPsjZ$Gs6D5WGgPHQxe*-%E0**5Z%FJr`f0$95E+2$RJ zQ7)}i+RnGi-2zMk_c()))6$W((U<*DAnyag&>RIc-@z~W{s5b|tCqC|VVz@d>+Hh9 z3ow53I|b9g#BMY0;JyALYcmZ+Lt$rz3g~ztCYr$%4FT35qqmA%ZB*U(VYvQtO=@#5 zl~}u6Y*Gd^wjKLcnTgJkzQcimt)?qs!Q#(jMR;>{yS==AFNJ%+Rc<^UX-27u$S94H z6oy@H?&WV=nUJ%p`N`VT2%=VBExL^;-16`fO)L^}my8>TB6#+qxe|5lo(dK({4OCB7zsPHeB6%>*3=kbPd3k$`yL4XPiERO3r&tY(zMacs!%HGyAw)8$D|ah?uuMhWui1$#NI7~)H(WIoTMkLTtCa1 zd1US0k&O_!<0KP}wm3}Zj1lzAVb;B^bPi#^A|9hKBNeLg*1HH%{h+r%Mwe}gZcSj% zhc7cLe6)Mo-MtWwLAr|ov5ClkoiZg=e@bS6knL0qp0BTmA5m*;XnWe- z?T=h+L^@gf@VNBJLj>Mj^x2VBXMyH6x~5r+5j#B$K`&OAUn_@kY~qLwr4nz)U=PDU zAmV*vwY#rWqSxzg$fN*`1S3sC#lYmERe3A}i1-}x5G{u!{L2t|XQYCV?}q~dV>=DI zq1fGazxu7{4WccnM;>xcYX~&~^5|JR8~x9_Z@J z*v#vbG>zEQoaY%t=%V4!P(b0Z0O!n4X`w&t_jml-=IW=;uC(U!afBBtC9h_$c5MZ= z{^E^4vw^Ny?V`SBf?XgvP$#@`0`Qrv!%z#hzIDP~BSgbT^9MosyKobZuw7>Cc%0Sf z3K8uM^!;VCo)}GaYQhFSd+C=!_KV+S`G2zWHXl+*YhOzT2Y%jvhj@P;?(b7J)3|lJ zV6=AClQ4w)hv|&Dh%>l`=7UhGah!69y7VA*fZJ{HOC)KRhmGH6vrAF`4<-K;oaq<# z|AI+6M#r{oyJOq7ZQHh!j{U^8ZQFLo$rDYzzxO|TW>4*tIl9-nzPPK_@u#jSs?L@( zvp>y61MoeENW6GbALbXma_OdF2V)LTknO|BM%QIZQMV9ku)>pejmiRoPK9gY4Vr>c zbRtZN)H8Y(9+|;YIPg=~z)2tc(z8x=(8yUv#`(?ZYf^X*XpL-%x#2&0)M zV{{$3f<@hFv80;kFkyk{_|!%AvO#;gN*SuYTUngqR5jC%A)n<=K+LYpJxV?V+MryU#=YQTZ{n$KrdLq z&x^OKBZD2mZ(m>}+Ur*TS zykCLxD;jtwe2gpr?{4xhg0t49dTI7EUmo=9e#t$x{;QP-kHSYfkyPw(n9#fQ zoQ&D^OT=UP^u!Dp#&f;v#Vz5;rf7V?O+-QV9!q{3F2MgoFZvbEK(nB*EJq9Au%7P0Fe~e}+zao}+x#`G#EpgER z4_0r9d6#4hUlSLE1~ffdz!OX-Mb(}S#`ed2@f67vgO_-UezBxW3xi?Y1Id))?>e~! zT*&KFyhf(nf;CgQVVKU$zSGM^OTVw{j^DM+;7EDJ;Hs{K?)i7!=q+ud5?$AOHPBzAtMAoX4tzcH$hWyhNBJXyF*SD{* zf;Mgql#6H|rkjcU<08xI<7cA_kB&?`8Qg*rw+CaN1SH5N(`~28P?dTuzZ?#%h}@^C zm>g4avq`@{mM;pX&dw5Y^p8O4on&3_tMBv0K&HsJn^ck}4nE`+G*0P@2|j|*Kms+$?9KZ=-F@~jqRm289GlsFfmDQS=2i@V z@1QN;vo)-4C`^SNXKxXDTn~+D5fGxGm__H|;p6AqW%(zX^$(fMgc&dqmXn4`bU`!6 z!E`$G1g?ah6RqMxro_bgizEI7Tq-`b-%OVF6C*fO%m~W@?YVf!08s%+X@a>kqWi;} zr~EwJrltkB9*!BAqzPB4zsXk&qqYJ0hIJUmLhPlv!G-s9h_8l$j4GBkw}5?3HnfD4 zPE~B<+)2izB69?9kt_Et2A+_&6}-{*nj7w2z|JR-X3clPfvmbW*4nNl@ zhrLDd7`lC@!ytR{VtCsGHPi>emb6;?&XF156N=F}lbSsc3*6;U!zxce0O z{5!!k@ckZHc3n~$gxAaM6A1A6I%0*6kJeg&GN}gH#T&uTURCgvOAw_U+oJ9@W_Hxx zvl7?7uF>EgKS*m82Sq#w}yLWjg@@`BeyU0=uuh}yMvnx^${+gP}<8LA}f zJDu3+uby@3*+DVE?ecQBqB>9W{5mZQ7J@p1w#j=eZ9??3sIdW3cuP2|K{C!dI(sCD z!K%{&&STA$*zfI$kWn$fneFRcP8I)oRTrq6O~tm`-1AoF&oIyQ^Jz1sa6tEyNa<(8 zh+GU@-YnetL^$M5RlQzfdc{?&+uvMb{&M-qvqDWL#vcEwkb7$06P^|4Plt1^Y1hDN z+eSPMBRC65EBV!pnAR?4C1g91`ZyJ{m_2-@9|V#t$EigoWASPtI;rxH6|JMA?w24f zIQIf6B`wy96yByTVieOOm{Jt;?rJ+xs#Lwo;)BB51uR_#I7?Te$1-_9-|@wP5j%f& zF(bG<^kh`QHlrI67W2s~{LFYT%SF1_>qHG9mt+N<3eh{_fEsB~VBC$Qr`zETUoJcr zXC$WP9GJyM*(S_yj7pyJ0huNNvGWSU`J+@*PZOkIm&yKQ;^TYLuG}Z*!M)GusG9x7 zmf?LnA}%d~X?|W{MHL6ILY#2rycow%wK&20NvUP?zq3v;?*9y`cK;KBfdJ|My{Jjx z{vTtUK$ZFbyjgL=CFzS21ff&>Um3r(xR~`$@L#-g0ENgoTx7v;-5Mn2JCG_@IEL8~ zZ+{vC6^r3SK|J!^>AS(xW2@PjA2spsY_~m4P#~5F{19B$^Q*;Tf>~cu+2??hfuck^Q$JUzcgHRPS#YY*p0AI0@i{BO+VKK$K?DM_(o21PF zNF}DlXr%Gfs+fNhnka+UDZfeF<8w4iCylyym$YB%dFNBxlyU^P`SDulKzwD?eCi*d zxGZb(=J+VNz6g0Ui4=4YQ2X{9n*{jYd~KAq^%dn%YW*5JCkQudEt#hk$@e$H-^cA8 zFe<|ALYLaeK4%D7?;+b1-;fRr{PF?NRw;PUNr#83dCnhvye&N*3J9Td}<0* zVC>*qT=TA;p&_S^S=q%Cp77}Qd&H=B1mDHwcB1?Z>wCE80|hEu1o92(^+eFOdAeUo zU`R9I0K~uukk2$K(Qfc#_@p{k;i;tFZG_P+wAwY`{|7BFw8$Nqt!~9W@Gy$Nyul~2 zg<a251@4{$T6q2#dh@RIo-}#)o zumG#-n3kzOs{SIHq*eNA%p&U3hle}D? z-PPevVA{YR>wDnn$=%bCh_#;+wT2lzQ~<+UcG)QKG95`8+dE3Q_A42aX_g|Ej;R78 zT`HU3s@G`Sh%W5Z0W{@%kx(WG{&%N&=v{Y+>Eh`6@Z=B<1fHX{G|5;t# zUN7HI9VX*P+csP--POAh_F%tCpa$K2Hc>3B_)k~;cYSek+1M9ao(X4#mk{!%P?*6! zLJa$U@}E;6JwnXu6`DCLr+7Y;-tYc|xxMaoNJVo7qSFmSl>NJZ5FjrvC@m-d%Fc7` zre#j4l%iN@&_N4r=_6*zJsdg0exAG59J66-jMI^`R(N<8Oi~qjrmRs1BEd?;8Fw`J zT3A5_p>4~=on^@DN#L>z(2?PSodL<@+6M^I?MPy`N-Iy}^@A!;2$+^9vmFB3+B@yH z?h)!x-QT2XjYJPm6B`DCAA`kEg3`#DD5{XRk2u0)riRvT9T6(T7C*II_k!7 z1FAbXir0mQPq&-K2?%v+tsiQWTzjfdk>JlL-&*8tIyd%DUF)$kdThcu( z|6Kdcp}7$-2?nf{>-L2)Fbot4#|g)d^NNM4+h2FzlR_bksD8c&b_gs^9N(LGo*HoScQCStos5A+BU|4T1gX1`I8{iW5^*1i9 zEAw^E`>K(EExJTrd`;Qmm@g0KzLqy#-B#*{{7-`fP$$vFDkK!YTtN~Akj&W>rV|PZ zHp}r_8CA1vJS7FL)^qSzsXV~a52tyF9p5*dBbJv_XO3YRagI@ejk)5`@r&+Srm}hv zigS+P7y4qcIH5{W9Wfy=>}jaFq?t}ayDNx>-otc>w*Rn`lNys7I8@d|yKRSw1YYi% zFzyn8Onc$2nnEuomxykVvTSJ;k78ssTe-?`+5V(o<=+jAlp%DwAZ73yE=_2ELe3C4 zT$IoSM#}umJIawV^z=piCponIr})|ZkIQvS{C^yG0p`P{ACdmejxop?y*vNS_N+-6 zc9sMLp3R3#JzD-hZU66Ic<>!1SqDo38J#Csvu%qKSu|S7GMej(@OJ6%V1$mZX~)b1`_?P+nHFU4l1Nq8@icY}iINw0vh`)A^M@@Q6gD%>50jtLlC;%-!>a z6U0rg@|sQTYrHR`jSI>hfH#Lr?2ZDm_gwgLb1O28^)38}vgUN_=KyZT=z_bGSE}OZ#pNlM-iGtfh zlfl=s3b*9RvtJeW>NWb#+lEhyPS6L+N$o>$P+knbe}>;RJJBff*IMAW8hvqeN!pd2 zJw0Y;jE5t{Kvi?K?E>HWKbO2%K(w8c8s}*w4rLncxo(_c!qWTDh_?r}B27FlIpb zbNPLLc63x+oZlPpd^2+rV{m?7C)i-@X7T-ewV-f3!k8o)pV0SR+@^zApUzJHETdV2 zGcA&TfbI{)N#1=3p#lky2*&Tol3Nc|G3h{3t8_ikxSr#b?@#uze2CTNUe&X7*QtWf z8IdFMjzEKKpj3J%ubox_gVIM`pr?)V|uWY>}d*w@ImTk5{~M0F;q*F*OJ4O2C4s}^*5b>#v*l%I2E z?r}}HLxnE|XFL9Cej&5ur-<@DSJ&w^$3E&PnNOB!v`BD;fIS2?9&t@262*`VJx|5l z7^!K%_ZVs8-}<&^a3DF6&o7~R_VnQQt1NQK9#py=G0>m8uVsWQq%OYeszm(b936qp zVSF_xt0vf>BKx&o0hJ~EJN8%2F{LKnaK-ysmiB|O5rR$4N(!tml=uFkS5&XrpX1qDOqZV3kx#8M=yNj!WfOuMV|u;<+w| zk^QZgG}6=Fc)?d-yhb{Z6%5%42p&X29K|97ri)%XT9fd)=+3}(WcPU&TAtq;kpQ-y zCfGt5j=hHM9o_9L9-?azOvPL|<2b&aO~9b0??m$)-SJz8QE%U$b(R%(5oT-WWhxxV z@|~Xa(FT-3nPKgQ4wlUx;AgNt8ue2j8^;;`Py-??YEYp=Au!q3%OMmdWa5KY_ zFTxD|8SdA4n}LuiWprsQ)FnESt2eB;bJl+qFTt{^jv>3VYw04hqvJPCjD9D!-gs3ilD!8T$qa?QuG(j+Bqg z-k4qdjbmzs!Y4Eei=N4gDZj8kv$$V~H&wnUcV7Bo{pZi3X(jcPeY(L$=~D*8fYp3P zx=9P0M?HGF2N`=`J!bbJ0(6G&+DOj)y$gY$q7S@2-e)9G|0+p}d2(UMU1okRVTqG& zOV_Ekm-^94^e(p#H?H|49D*I@*A2|N`%~eAfjqe*nhW<7!3tni!lRY>bqZnq?J=^3V5o2>h`Vex)BPpcc7h;5%jD9n*s-}M&e zS{fHm1P;!GOwHG@@n2DT*JC;dx zMBQO#eLm@4z#nCaPc5-*BRxUvaT&J4EX!O?` z01fcnvEFktaO!FVOgGjnK@I=mkKYQ&Kks4%-x=)McHQgxGu=$ng6Pz)DfKCPZbTK` zbhVxT7tgMA&Cktm*B9TOmt!`pe-Bw+Lv<4ATDZ-JMaS7cBEGxSi6WWdxN_$jl|R_g z*mklJWf`1PuWWK2@7z>59t>k}IT6|=Moy}b8==fTRR6L^D*L7sc5D?g3*3c@kRePu^j!bgtUrjL$c29^X?JZ3X;GB17ida>rUS)KnM4)FS zKcW0QIIWP_pCcyrrb&K0s5bR9^cl#_B*>L=;5(O|VO^t&mej>$>L~73{J4JvIco7y zlujx??v7I&aqq>5^un8oz0$F#sA%uL|zLrXZub5mT zQ_vmb2i8^7s~pz-z+4xDXLZ7RWsqwblPY^hQDmlFWTSgfV8ZRUfoS@jje^xR*C&n3 z-^bEXf#CZ{WuP3@W8gR%Hw#tGM>C|0Z-`>CFc6=8PIKMnLvRh~7WmMqYabq%!t82H zD{4bmyca;rbD~z(FG_PWFGzoV;1|$OXqs8H?G`jl&T5)bauu7T<@J`2((sz3<=vc( z((tx$sBZ*usBc_Yo22D^6U3{zkJ9k!*Z~?@)i>^2{!5Lu{`V>W$!eI4PdJyz;qwMG zHtqaxNezek#(i^wLJxNJwd?=-E&dB*S6|2eSB5|5=_E&eGvkDrLNgrvQyr?@$UnJU0xBTcVl^2LT2731T_I=>HG~~=V=i&KbB^mN(7-P zkilJxJ+^Z2xs#hp*W@s`VM#5!TV%xA^!+oaTK6p-e>^u^1Aj(=0AB+|O zA1E^%+MO@T%hJV%uofp|=AraYIH1)*p9^uz3uV6)fkT3z3Ibg@>}nB9sQ?9=D~^db z<4vUxt25I9d>u9cjqWM*(DcvUW1p*aqXX!H+%qwA!06t29s;YnWFvn0lGk5f)=SB@ z_Jone4hH8Kv9?v=nIWK7QrDb+NRw@NN*C^TGUGAsi!d}pw%=U+>E%@L&b#u7#?Iv1 zptl{qYKM)m+;jsviN3%#9-Eg$^%a$f=TYsLWfF>t4#Z*NFX zRWD;Klz*!9Od4EMPxQ2wrfPT?y*4Iy-F0f)81d@V4)))L&!hge6=V~qloY~6-Ie&k zNIcEk2*S#(DN8qW2CICu>c`w#OY4U&ah$nnuNkdiFpOdiIFjy&Tl&5sdPw?yxkTlXG71{5krsg}5se(Fc^` z7sj#pB^-rc6?>T>mlv^!A5)cvs3?V%E4AJe>IW6E=$WP`O<(^V467CcR>vO`F{c|M zP*P7p{!Grdv-(Y_e4yZIrxv&v>+jr1gx)9OwkAVQw?1iCDHi zLHz!Gde6L>zr=9-5n-Bch0)gX@U#_Nq`Pc^3 zL5X~iowpA9JTstByjy}#oB?+a;vx4lVv2EAljYwXZ;Xw@Hed&aY(peiGNoLd93~=7 z!o-8Cs*bHBvc!P{vnGL#T9;ZD2Bn_>FaSqb%_5& zIVpBx)CvpI)dN|)GEBotuKfN*VBuG#R`eCZnwj9_FYW5>=t=%4+DC}Er^?;kd1p*~ zDyAb-x}8<@(B-`C>PJJ)94C9)QR5*s^!3>^Ir#Z#%yj90(Tc6-#)Ot=$4=l;9+Urd zD{Z!Q{yUq2M-P)l36eUz{flD4P@?S}&q)X~RjotbNjy{vyO~6Kz62l(Q;CLx>S#qz z!LJ%;Qg?fkmGnSj%41sRZ9X`uFb$11)UbD+H8vYY3d||k(`6DH#_^CRvF$HEpTOM8 ztuR|;Wow>vQGmAj^H8PWmMLK)ratHI)}oZasg}2V^#8DdwJW#EL7(kNMhY0WPuu0d zCiDBcK4TR_8TEH`vCM#mD7_SMUG#KKo~sznbp%iAFG=AhUfx%;7P#{B?UH^``LIg? zB7DO8Q{Ic7Quc#COa-)@r@(V!&H>ulio-+j4M(BZJ9!S1MH1;v zkB=G8|5>P)OVK?yI1jJ8a=g>gETZu|C5kSnNo#!7WWMYEa?*N+J{Z00jGZDh1S{>d z*HcUyTNohy^fkG>(X3|2e_Mp{3;aaPz=XhXm%eg@dr7~o!-2gd=z{i5J0HY3z#2U) zNGs5Bi%b`3o=DHv@{5Nr*>)!jtA z1@HKxSGCG-b<)*YG}B4*8D#M}1A`pNapDJm;)i3L;5mDicX)RZ!^_`&J)OSh23}rv z`jyLe8?uOhz@HjTEX5Pria!h>kW-;!M3Bf1g!j;%jX>$bC+L3A_-R80LbIs-c3M2r z!TEWA!p)Dt`ALPd@|&H3{2PHe%j*0POg5m3A}OV`89SCDa`*onSr$;)nMJZIWaYnI zJtP12bpO$$>%0^LhPecHd5Jv+m+s}`YA`b_CyTIaITC}0yUT&+IS`3Kx>lbdQKDxN zy!x)6+oZpuzo=?k?;*@Q-mXGqhyWb}^%m}ASi`krqg0-a`!=;u~mi9+j|7K(=VEq^#qO7YToQN~Z%Sb`IR zkk@x~oIttG4tD-I%t3&eqJGLb$VJSuL5i<|eGTeq&~Ay%Y;WQYBhScz;Yz*|iRZIe z?qmy?CM#zj_7^CZ;$27m+ZVmk4W-WZROXAXS{i2-^p5AQk}tkwtCA-r#ZoKRQ=_!8 z!l~dRPZKsixcJgmoiltX6i{+hgLU?kcqAMsLt~CVQw75t_H8dzRb%(@+qu|-@fcNZ z+cLeB`@o$&!QasJITrh)?ixp0rD5cCkcQqh2@?QXP0T_PJoj!k6hVXeLWfFnN=*mn zp8D&1<@ZD&)6eVUw|_6;@w>=-4|Dlz2A!40Md&_(NgQKR2gpGHm&ARnT)+umv8Fpd zkYiboU^Sm{hrPeFO+Q{l(D&@TkbKVe8x_;;(2xl;W8cRKbML4lJ6g3*`jjdzN31f&A}1dyrFUB~hbB1mgZD>8ulUtpxviJP%vt zb(KjZ2p-f1Sknhp1?OJ00@96iUt{rBn_D+44SMc9Z{tPg&#Z$3DpS7B!Anyh9(-r| zKZV_Rv{%_~$L7fb-^k=bj4h`(yBuuSvXV2d4`UqH{i)Zj}>SAl-_;1m~*rU`) z2BqSJYom`2@vy-wZF%Q>=$c*bu>0{Pg@ZoSou#M?D+wsqJq>Nma#5pfH@rss_7QM= z+j6Y@0+)U?_s;|N3+Ej9`H@d{jf_#I_N8oWP@Kc5GXUDE8JiKRkcBGc1y;fJXtFv{ zVRoL-T>S#I5pVWITOw#iWsYa$qNiHg53VdEL6aJXg-X zl*Y1FE?*E&wVh<%&L0%y(U=jGX3mDe)=^o?v&qO{X%Y^|mNGC3GB=P#Z8J>C30?UE z1{QK0f|r5wQlAy+U%?~=e-Tt>fhU3H_T*&zt`EN%aTL1HZZM#C$kb$v*E_<_5_{8{ zs5`Zsr8dRt#$B1kT#ATfqPb^)o;dvFUG)~&VV0s%LEmhu9B@f)AQW&ExkWCm%*dB| zG7hwwS+jV;utgVoWT}`BS|Q+t`x@i5IAk_h2s*ju`&Nde4C>x6-=i4 zPKaDjLaFp>9QV8A;FIXHUCg@O4afWo7m2g#;JeTbdfl$7O&d zDLNHeGn*+(G_aa~kTc)|9?-P>>9wTbUXK;p4`9Z4R>C|)5>B=+-P5?KXIam7D|v36 z5uIbTsUcLI4mM`0RtPQMYQmC!87>S0`|AsGlGxvS{Sb}oEX z(ePCwOHtL_wMwrbA0``tRj{^rUdGa074C%D=Q~tbX z1Oy5D&xrgK@g0Se5dEos$RvMdZrKH*=gGeW>UZ~c0LM;pV%9i??B`{8`pJ#kg?{8O zPf(2P=Vh7`WQH9>zWo;F$cx&A0vvqWMfo_-$$MtV!<|CDCn}txgtiMZ4(}3VkXqvp zkr<8vw2~cqb<`yHBgJI@JO@??yT)!Z4I#bAbB>gURP_s z6-c**pml}kcJ=uJ?t!O=IWd?TK0)xJs(jYc>Hl|-xba-?b@b_2y?My~RNRPq+d)@t zK)$FDfcFa`lAj&4=mVY9{hAjx1!N=faD!MG46S%&F53_$5w9Xt`yPepGE{+;TD$;h zC^6(UZokLKp^QB*j-qUKqpZvQv2ezUU~`#0+-{|fhQ4bRO``+WF!|7CB@Ju#ZsIf= zyjt4!{hc+^XLPnEo_1KTonxpz+FM9c(*hByh(`}sGsk|+IzXt{3T>iYb>Al>xPxPP z9!H7;9?!*YA7=z~LKiobE)Gu{&Iu`u-i4kcAjwL^8aW)fAw(MfHeA#V!F|qAz*Uo( zB&DdDP9#OZ?PWUCuZILyM6pZ&nsO!}81V#rC%rE;6yc$KXz1tddqnX2@!!9UtxSIX z9z@+;vN=m6<`Xn5g(!1GX0`)}YKc5b7{<(C4Lj|vi)0x$C#`SRvx^Zkw4KdIy`+L0 zj@UvCJmUybt3A3x-HBv?>_VKw6Xd^BTx$-R>gHwmhb!xu5-s*O!>?hNi`upzS*u%~ zt|2rK(fRnjYar|dM(J6ndRfz-vkXj2JM$OhaNbbbnPQFmo_h++wV#;~uT)DjLn=lw z<}Q%MfmAIHSrl`v^blfYDOXc}Zn+1N)6A`M4bzd4)8Vg_W0}T9ji4x1-F=9XJ66I- za$NxjNlu;)IYHMop;g4B)T9Ts)Q?^-l@5|G#SV8H2KN(Z@SXbY4@aT;2`v+~A`~f5 zOtIEgOjw7O^eO~X32Hr+iQ-lxQ*554aP(rEUeuj;`O+Ems#&cg_{s zN`E`jZ9P${z8tja@mJ>pzTK2d8+rc!g@w1I8ap>bq~EhKCWV)5Xa0X6;fG_Fi^~C@ zMA6Z&zB=442-M}IiX@HQUOcXu7lFEW!~_r*y9i09HSr^%`mfnIs5q+7@rJnckO+SY z3b4Bd)&-Q-m;VC@*JadQJf%PRqLyvJ5c=)w|?E z8@sw5kg5uNl&b#rq`Z1vHev#w7!|wU^8Ww`Tc7lM*uHl+hP>SM zazPY4iGrtP(@z*|80Vk_CBsC-{Y6)co&Y`k?cQ zfyp^s`yHOUj0~wIw~}+IG7xghWXVs{w>V zNV*pA+I2V30|MPB2|_0o#i}yR?Nk}vJ-iq5@#+ERJ_Sq4l1Pv4ilBLWWI~}g1EJNC z{qXgsl}KR+g#6T;C^9UT91cbM3{T1c&>h>`6zw}Z>n&;D71*dJ_{!Hv!n!{~?Kf^Lej;K*FgLF#l3wF}$(OaW6>3q?j)zy151VG@Z{q)w5ChX7wky)yqN@Jrdv||rwlRpK{Tt;b}6ZS&qO;r{levCJ+D{a#vkFbnUE6$W;O}K@9 zA+grTH;_myGK=HBgXK3pb0J_?n87JCmsyf&?R3-Wa(=c4zQV#nRF4BQ_9!Yc7|i5m z<@0m@-eG$t%f6MU6o2$cTfC6`<4ySjDTd55@;hm}DW7OX)4-=Ah#D zLQny#ghsPOzpkH_E@JC>cqu)Ub61&i;T{N@hIXdh8;OHUwI!dI(;?R<2I| zoz6Y&N^81+nDWj*YlX%mS$XqcrPC89E5V-;wgotldKkPJYgL+Z@=sUths^I~B5r!$ zX$!e(u&rdp3*MQgmyP$k%z4b5Rp~m7%yQRd{h1x|k24qSCP%ll>W8aru7sW{H#KV4 ze&xG%ryg_*$f06oOl-P_5y22jooj3kbXmvwWFjw^*EtIpiay}vjG35S0rbQ@BCvlK zf`5jc7gLop>wk0UvQ(|(=@KJH+4Xr$PCd-S6{e%SwG7So3zXC_vE=0^H=7dvz#_6D z`E;+A@xn-LSFZN3_nf5g#i-2EL9a-`j^l~imS{mv(2wUD`jA|1hPW&%ho`aokV9F^ zAueU(m9QuJXdo}xW&o^1TB|$p$Jw-O=K9_trL;-&`DrRN%WG;aIw5|55rV$&5fB_p z%0Ua1$5C=0-+l4k?p<}-1WTYNgC!Fk%@>9G!va~$_LQnp3 zmNaf`AK>YOsU!u=!PH7AH%rlZti6nH%-@Ssh6K%15Sw{@tP8`xQdO-f$< zM&w$sw?k+ z!gqq(jpBr=o_K+(;)KidVjf_{f5$2hd_PqQ!WI3y|E`mL|7&1vq`~)7{dRj&LeP~U zTv?ECds4#pQ{B1h%VKkSQu>PaQ(e#j6rV|7by{KTXc!uvW!=L}s8Jq4P0(w*J}vxi zc7)@Y8}anlHdrzoRpi1y*{_l)oz@}kv)GZK?Z+~aH1cF_`X0)9QrU1QqW)o?enXXe zi&8cydf(-oDbMXv&n3#hkp}=#hB>C~hQ$b)A_FJ%`=T zsjOeWZOUPPPOmC#(-_NKQD^o%hqb_nSsV6y>r2bBjF10lim|8|2i0O#Eo)Y!&)?&D z@TgE#6Jv6uhRzV2YPSUOD8jIU+E~91-c{|Dj?KoB-Izq@fU0vmB9zjADH=&KkBmhM znJq7qjfirstSVeBl;3IzV_Q-&Y$~{wece#>2UhlEB;1ViF81eA){ER_t$UYkN{pgS z;fZAEhaRrOx3RR!JS`muelFW08y53wghMcakcbf69HK1@&^_!@?~n-|pc23&u+<&1 zUeeVTDda`rJb?DjPSZufJPw|JeDp@D^6DI|&$9uQMK)IQLFm_K5IvrD=p!q@Nco0W z{()=Md(=2=)H^CValq^w8}Wfz{*Hk-eCkV-e`xZCO1PcDwSdPm5b>t?JtQFlDgS_7 z285%0gH`zor4|^Bct)f^bc>G6FdfM;AILH7$+?CiOG;E+WDLe=-jqY)oA{!0e{EbX zeg7&0e$MA(-n^&=>{(Fu35B_7cbh`&=#Ay+^HX*3zhYdL?ws!RWDmubx}} zR&`7>q*bByNfuSTfMUi@hSG>(D_n59xkzYf)tc|Jj=@@ zld#i>b&lW26g&HhsXzRuIQn-2);M@+@gn;E$TNB9p|9REWk2)Nfok{DH#jiE`_WB| z)!l8L+O$m^_j-vy+`bTQl%#>gTcaI~N)lX~U93Umt0|>BoEzIw##5x7uSlLMd9733 zm6|q+QZ%us^8RLp9#lz;SqXJ>!c^I}CPLKZDRXB$X5$|&SgJ{E?jCAPmjsoM|MZA%T2~Z3kee%4XJ`2khRQC~LgY<@#l%!vbMyl*SVf!_xqcYysKgXn5^r3a!TT_sFRPS=MfayT*M8nk z4;kxvu??K-EL-pIjUWp9RRT3^&*@21*@AzXn7`{;C0O_UrAOD}r1cd;-IWh8JqL$z zBu79x{1pb1k60jnAN)S?N<1NYyFb~|SVKzHQcrx0Wrq33+YJilBN3JD>BxH> zf79Oahg74==yIZ|JmB*X5tRE)DZ0PCfW6v;~IevhpqrrnP9u!ywEDpwDw?R>_6&gJCqefB_WK@rXyAOh)lrssV zP-@1k1*BE^n{Y^Ae)RgSskM*F>upuhUKo8LOK z3taM>@mr%!++g1bIi#`}?BCbornlKj{h$^@wY&|p{Co6TiPle1=WkXBc0*1{Yk0n!1LKe7s$-;g_YqSDO-{A~1gWoyN|GoBqbB+r8s@0i?f(^7<|7d-y6ln10Ph#t5o}{_XoS*w$jgJ!?ovnK;L%GDSZ(x(*%indj{qMS*xs4GM&ZT442-lo}!ZqKM@Z!t6y21}_xrfQc5G56a*bYU68K@>4=% ziNS=ph82G4d1KfKylJhxXf%X#hDjP1jh4h)OAZQH!K$9whCG5m*)9$lBZOxc7ho;K zY)6Y_Qqth%r`2HTTSZ;!;61F8d-W=N=Y7Q|wd3=r+#YV>8&DX72+cBCFKU6;#^*kMjkWU;N81aCQ^pmR ze%kteZm);H1U_Fr=g#u43Hx{j1$jRX9P0Y~1D-GMK0lWbi#B%m4f1FG4eP%DN*;fH z&i$a+uJQxSc2a=2ne>d)B6kP$U(neoJCoweSn!BY-li7oLj=NV{z*sKCH8cvJ}$cD zeaUXthgh%bRSOr0dX?!pA*cJh1L#RQ`ZhaKF(|u2C>*8Ft_V$SL?a=qn#&t%^mb~g z_54FP_p%W7z4UNY(eOT|_9CQ0BRcH_;8NyP>P-pQCLLDCXI*M4bXZR0&3&2O60s#U z0E*w)l?V#t&U^#OF}ZsW-GyQK087++j9#{`QeGtexIi9h+-RkgojOYM^mp()^kPaO(JUZZ>tfq_sam4 zfsR@+uSYX};C{g+W!IO433rL`C835gBETO;r$$p<5HY4sqRvD1EPEZ(#zZE=Aq5r)_*hi>V~b`UDdb1SXm8S-7G}V+xx&gm z?>xT7s>(`_S!(SfyoM?KjcoG4cSmveZW~+C2ku$_@O`LlLN>2Yp?a_%z^6t#Q3l@S zuiIT@Ld_N6OtPW2=j$Hk`d8C1u^a}BVnY)#_C7c>ZKB$4UPVt zzNAu9Lb3cO!&=|1!1BRII#Ck&X;GMOqE8w~20Y8*({C^#;Rc*?@OGMt(VmQi#a0Hc zCywu?gSKYFq8BQ?pyDw~_q|3jb~VnM{}%u)K+?a7&gP}e+XMWG9UAzGeXQ?gDud(e zs^Yi2U~BeDWwZbZBHdMVF+u@mrILlxhng0e!h*VW$AVpHO}RkmlF|}Z==0U`TS+TD z1J-{Vqzl|{DP>_&I$^n+c9~RPA^#!R-9Pl36j>E?9S=4)X03jiv(}|NHYhQFq87mk z1t#A*)cuYBoIE>+wM`Xq^utEUFV11hzqq*i$Mt_)|KsB2>&rLfjjWk1wdtgd%-MHC zGLgQVV_+fG0_;{0qc3uKr-H5wLg6wM*$YSB&OSUs_4)4eDT9s%}`0Ul;|?xdB1Rw@rK>+kV#L2VxJ# zi-R9g>$QP1DMdKC-)f2a`Dy-E6uWm))?Yb1jpUW{JxZiKH!O%r6ll-Ewxc&yc=%c< z50`MR=A%(ryS2(8@;ttDYXHISa!e$||LdG*ALK4+v8^3_OH4K`xs#vg9Z{H>Etsu$1wP(WK zif#Ko>Ak)6Fx>K6#i~in^4kYymcB@**Lauv;9}jfn&z|)?-XRPuF(I4>5i(RBWyM_ z-9eghHao&fBWiez&HnPNRIzWkXg*?c$}}^`^(%OkWOjsLW-$^qx7LirV4Ta&TA7WA zv%p^{hMBPlpRTCL<#w-;S`cO(knBZ=5v44>qX6)DMxcV-vIy*FA1Pcvp<7|68c2|W zE?Hq>A~85RZX%epWT!-r$zoa-{ySMP%tx`E(j-2sKEaJs`lQvwZl6Gj{+#;MM9$v5 zKIy}{+b1|NKc7A&8L1S@GPy0UDydb?NoRn0v~{2%Zp6gkW8j@#LbURd&}fZJ)n=mo zABi#zaDx6}V&Bm5xoN8Ahj3F9*=L_chRrw+$UbF83&0(ZPbTS4X#e3hIS)N&@LfuT zIzfq-}jj0rSFSUw0VX4G_zjIyhFW~1{~A?2SiCy7yu@~dmqzK@)ocgN2FACL1|Tb}kOgu@PQI~YAAyk87tna~_PrnY zPQ?zR#xQv17vu6L*dw=a&_sE|MQ-x^{Beo))YZK}p>jdK1NeH6NjxVX1aX5$<_S~Y zTgkDN8l{LU`5nrr)<8E8TV$g|Z6o^|=34>FP`7)oRq*b8QL^4x{;YV45f@F;F=g1~ zA*#yv_^6emGretUF4oFdS7^i^?_2GM^cA}*U1gSinLct1_<7u6ePT>!h`NOj(R6?I z1F_Je)I%JH>#d?MD&38m)8e`e!;_Oec-`067v0A}+c9}A41Omx-*Tm5VHf(kW6O<{ zH{>i>8fUKfT0Jp~eXe^D2gc^2a6wI=>5AH==0GDe*9({ktRWHS7CT3wI}BJiLQnOB zoizJX70<1WPS+cwp!RTfqa7a{Gn8!~4qOxkt&FK@;G%zRpCdWL7*PgsMszPmE#*A0 zR&R0&kUfU&MoeC-D{_wUxpD)OzaH%0-HcQa<=@YU+SGDGZe*GL6A6lq5-m~GQS}LuY z#8ntGI1}IKJbAl;@!346&k=jb4jQAQ{o5;kH)xR7w{(rtDT+wfXdN?5r%)U+0p7y` zNFLV;NQNwdKbwIedW#=8oEX&WhZ^&J$KI99_acx-J7I7h9h874)(L5pDi3hc)c?pM zf&Iq*fkDsRmb73%EnbtF*@g$?Ut3$o#b5s?un!kxY7z3T99d>EQbvhpR5-ZiT3VBf z3E(#Yuv+J)72WE?MiDc_OSa|^=r~H+ofxNHYQPMoL0*$BUvE&@Boq*rh;@Dzl8%o8 z>|j}uJ6dh^iAS>ycwea?c~e2nhnmrRH&->SwNTD5TWNf~LugBOtg>@hYsj(6&aui) zbVQF;cE;{HR@pgL+1bDESY^k%j#YNxI#$^kx{g(L##?x-vU9Amvv(b<>>R7?gsx+i zonw_9a~-Sf9INb%-*c?86C(Pn1&=;f+1bC2Rd$Y5cJ{7gm7T!#>l1H|Rd$B1W0jp_ zm7Qaionw`qW0jp_m7Qaioq>CTaZNcN#kfm>3NgX|ZN@8EHmg-&jcA4lZ`})4Xi>oW zJ|tGp0F7jP?jfLkgse%)1uVIWDFD_G&6bP_B1>i*YLqB0)&(Q9u4%RzXBFQ=fIJRx8!2kO zQFX=W76-awTgvejgl3ftgt%^-qUNQ0IZ+IP)bWgT0|${PZCL3He+f;Q>7EpQT+tao zU2n^Jhmmmx3bRr*$qLQ__IYd}V(G#A9s02_Jyxd6A_qyltQ85u53~)ySCgD8qaCRk zp|xdxF!6TWVd}@@PVk_@*5IxFYdrMCHbBp9w(+f1(6!qLvrYteD|?xm7&KSjRvaY> z3q`P7y|!7#v{*p9JJP-I;Em^#N^x>;77t;mW)coqaa%@96? zqWM87RzdrXT4DPUU>c@_b=8QB)~yr?>1%vN)(}W#1nl2G{kMN5%bh-5M=5Hdv6eM0 z+#nWAto0lLs{#Dz4sSn$9SO~;D2 zSV3eX?-1{8V1>mq1WQ>q1zL^2(kmNAtWj(~PiJsHg+|np?r|RzOJ|!XO)R>x7HS?g z&Wfx)`xPyUCCzT$%Wvdby%n#jN><&r7gTfgO(SkXQ2A}M3b{63TN|o8G@zJsYz~ni zAMSRVedMh=DhSiG}S3s?{97f*lxN8BbvR^*RQAs$uDkRbt2)d%L57KPTb zqDIzQ!o&|P?1R^JowkINMauW@|3k}-<6N;;tXR!3L&9llr*H_J8tw)uDf{4b2cIvu zvS_yKCA;OR#pgOtSSw4x6`&w33z^-J-27MvabDWuX%Af0(r;lW^kYlQl8ZIQ%UARc zn;X>cfQDZ0kh}a~RV|Tl=qMlPSNo71_7Xm0VD=;oD$-_fxIW?;qsIt#=mV5*ObKe~ z^iTxKs8Ld~N^zxG`FAkGP#7)J&DLah|L&wzqip9Wa8odt;{i%g;syZ7(6l{$C3Eu- zY+8g_9T3ve;_#DM@RUvcUE8Pbp4Q*EO^tU2ruyNxGBjY~PKG^fn$kUCJ3{(xtTzPg zeipaDNZkvATcPK`5GJS3PakAIV`nQ_(si`1K(EjJNBxvqE&RdnVE&GDxHL3a-U0vs zBn3L=>_FP8KG9kQgxt{`p_ObzzCa6sGuOQCHTsPd%+*qiQ0-RR1VbzrAK^<2J{|+-`o4>qa_ko*!TYewR;9$Qj;wipf@yK={PzHr;>qv5WwE^^)aE zs&#pkU_xUN6=S;R)A(5Yg>C39gLE)n&?=fsRZDV>eIv`JCQRJ&N{THLHGweLWx@RV zwEuGU-Nifb5y$erFtye6zfrdEmfs8SWr+bMWMqnMAN2eHPqcl~t=Kj-)Y6JjW*@Vn zQT+BH_(O^ep&7t`noweY6jn5Q-{|(4ysdYtzUFjlU!kgN0G+y+YRf1Xo%#w1< zjd;JILaKL7F)UkNxVfxYS@4XS=YOo>xqnpp!GD~83^FXZ|w^kznSBG<<43GP8}9kfw} zb{|45vpsT5Gy_!+=Hz5QS!}MGwc76QmpS=QH4r@&SnP#de=`gQyVpHLeEm6km$PNF zzboh9<{tR^j#m49o}-0w%2;f2hdAM9yR0gM%v(}E16@!n94RRZwhc1PvX<~ z<#jD9x@Iq62S?WY+x;int*;D0qVRWpf*_|DJPHdqK`jXA0OhZE!O%xt$U6uc&B0;I zWyR|Oyn(RE=7x=ErF-dLH*DADFlQAriJW@EVL4kH%W@nX9jZ0hr*GH@I6_368dvgr zp+{aMtx-zq$+#-=mQ}Z$-90f9>|8UUIZmj00@-{|egprEbou?;mv5n3oWb&Gl-{VO zw&sG+l7Gi4#if`5PU(y^Jpc2s-!U>S(z>?eV{;8c_s*8BL>_{(0^C!2CTMdQ6{W6XW}{eE%|Ul%W~1> zW|}p97=l%RUd?Ry)rS+*wC12(d^lN*ki(+;`Fy!cB3f2*t2s=g<_Lu)1GKe1LPeI+ z+&-p;Kf0C-zNXu)2PCZ9p4COK4@Qk`OBoNovt35x6R`hW_#F3^=I{ZKDu9s`vsJG? zvbQZz)NKs<6KFpAEJ3wsJ~)LHdSVzeYbRH0VtQd*2GXtS(vCjeWs5` zmsG5-IJ;$pR!d%2wAwi|MhMi{!JgGy>Yoj(c3{PX{Id{>DMG=UT?r`*Tbc7!hf(!~ zjbT}%QLLJ;8=kW#-Xw-ZfYDQD%=$wfcm7|~67)Q{Ur#r3OPF&LK-fk48+om))>QDH znBj6YTgyuSd8RU1nmGY^^V}O^xFHI zLSjBU^h656bG9vIt)(GcRK7SiYQ$ba!nG7bCg90?ngfjUinB_gqHut4A6PG?tmTBv|r0_|D`t zyyPKupJz80fW&)r5VZ|&UnW)uso zfVTlFBWr0U8=U7@aE_g#B=?k}1+yMSFIK0gM;$B>`Qh*$gf!FJsFXf-t4^|I&}X5;*{@+n^q+#+&u78A&;E zMray8!uuE4*dNJ@Z!QU~)(!gj-)es4Ozzf$2hF~Hx_I_%@pMoP%E^YWH$#5LqapNY z2>tJ72>oYgmqvI;Z%bo$mGwJoQk+$UwR^sjL$SstUP~J8Vh=3f1$O7p=lCO}puyT~ zecSt+Vf3I+keY|ssnG*t;RZLR6SL&D zkmUoZxsVPzFo;)Rk2LW(78L+&xJGkRPasL9O}ZMYQxWE_SwZ<&Tyxh73lZ!JMCBGd z*J$rfEp4l(jj8=5%NF(lBsnuRF)QY!S#-)oqD^)*S7+agI|v^DjefZkbjve4M;C=H z!Z$*@3mstE(MHPh?Y61WReg!Z6p-eFB}2@iZ>xq4W9tUP%?Fno45lCDCxt1sVoQ_g zq3_&1n)R0wO&*Ks-4|L%k|jFrEM>kJSR;Y=xFRuEGIv$D{5dj}(`py719`JZJxI5e zaZK7uQY3y-aA7R8Fc>r#4H=9D40aAGy`kZ>x?#JK=?=7JpA^1)ONWqJkX_y14jROW zO2dXq!h|YmU9(C&C;$8G!|y)L&;NXO_QTWpzy1ApXCD^u;kW01KL2F@_}%&W+1U?& zd;RtMt5<*L=bwHM&GrWW`gHaKd-eB3BIoCS{+E#k9faA`l{&w{Bcshr(R*i0KSFR2 z#dI3Ef+96d?U@$GrUEN&O_p0bijeVN&E3mdM~!-Nkg3+5kjJ(5ORhD8g}=AjOeosk zY8&O2R-&?e6T~7%ohu@@ysnLn)ML|Q!OOhv;HH~B0a?ft9Oi8q6CY1Mmex^AEs5}L zS%7?pIqX&@0=LVeZMAV1vyT~LxoN`KvGO@>uE38KtC<+iwQ^SPFMhvox9qC&P!ZRf zNPJJP+E9Qyfiy7)2Uy7}1S!K6IsM}In^O$S>d7A!TQ>!*^i)>}gF=2Zn`Ns$sZZNa zn@{s$Q@%hitHGz)Ib)Cx9gvNvYYjM z*KXYNJMVMuecpJVx87&vf3ME}e7?WG1Kg;RP0g+d0c;&Vn*J`TL1Iv zBUh13H9eOvB~x!?4R_yraIk5~Lt`Q}jDbVzb6z}$?dD~?da)Z~uE?bzva(~~GH;Y2 z?*m8z;z~G=?$U%?dh~Hv^*j8L^IsJn@=i0b3Fdr&Um$`X3NYoiv?J9cLTk3(v$A%t zEvwcLCK^IrORC>8sSIkUAVu5`FwSb`ZrC5kP5=kOH@`^alJ{!l=Ua?q`hPWhhDG%M zVGi4ZT#VaK_2oZdGQ@o1ny!}i|Jm?cT9~*PDhNXXeuSu8?VXl0a<`EReNfKwbaKOX zC$r8D88=~`T#6G1ao63H?wkTC;sky>S?nFKV+w5e*gaI%pXTq-LRg~gwRgI=UOe<* z^g=ou3dpc@i9UMZ6JQn2WR-KVK1x(aiE4B>juI7IM~Q0mI!aXJC{ayq-%+AEN>mRf zQ7M)+5Sc4Q%|6ER{ja-slZ`BLMF27#(&Q~QLhHKXOE6*%XKrM2yY1pUvs#N+$}p8) zJyM4taRqSsej}!eU@P@?C7UuOSm6S#_Ou|XDXlfWAZz`+2{*vA8&>CNr3w$%7Gn2H znJ>t-+&b(ZJwL{B37!30vt$Ju$#MsZ8z}Mu3;0eG*2l0keg{hwEROnTzXrq@ANU~1 zGhDgy4r0N-zI^EdLiDp=UBj@EFYqqK>NU^Fk_XUF^|5~@*d0RXIXHkW?vZ_mYp8kA z7^cvd`2LbrTaM=TB{wqc+jpl1skOSBS9kOId~OOGLpg;pFOEOr2yiWHlkU?SVa){E z7prl6S}Jy?e+zx0Gb{DEfPyy3PY)90XF$AyREvPi9txa_T;7evd&H|k#2k%gcf5cF z7D6@dy66TTb8$MBIYf=X3pez3Y-)uhY_-zbsT72j8@6Q? zExL^|nBLD#Wa-R=0dI$caEXzVK~&3mW#w*nf-!RPR=kt4e$5penY}4F6wFr5s`a%v#vDINT0q{ zOt~5m_5gFmT_Cp&fR9bzx#*v-3q9Ha=E@BOcmG({)3|?7dZPXqO*^LpZQ08|H;F{F zj)yZH!UFw?e?CEO5JOb@+BEfMTTA4Ao+>*3v3Dy4V}VD%1}F@p10} zg7|R@zz%R_gw}Q?jh$kWFrg5ss|N{Xn)U7FOY)SQ0ZJY$krm|yX4ulv5^Fu^w3Q{i zaL20FC{2k8A(2O!_K(Ly?%@F>?}k7`@zJiJvL<4HwNv zEP|3^TGmVI`V~BcB_lr8Mr}a7CO3MGA80%-75K3WBZZJmMJHDuprRrlOft|{zhy4t z(3_v*A$q?)j)y2yLBLG)!>d-2H#_yiK95TGX?tzgr(d`2a2)+oZI9Sy70WWYEw3uM8U&o$ z>mJ%l;31Q_W{=#xLLdq)BctXNFdH4n1xDqt`|$>u(PAT29UPCnG3h_nmJlUWzZJd< zW#i!mwluicfJs4dsuorfRnhlYgc9yE-irWD&6!(JPoHir0p_?ZX-=V2Oypr-LrLq+ z%=pU?^bZ`BDDIysA4Alx6Vu!PQJZemwgyhJ74Cbk$jLW$j8DuxZ{EHScikrS1O6ta zOYX`50QSSOuoG;#TL|c*zu_V`7^8_JNj6 zWzD@qaVJ)!5<>b109 zRy&o|W3J_{HxTaxFQ6BaoO+ zfn(N}00W5|0{4x!zzarcPByg^T7NxMnObiZeNi#MlcU9T$+ASz`Rmu$7u^SeFPR*6 z2EP-U8-HX-c4o_slsDweCI_y0y=j(yoSX$WCzMW@0-8TO%P^__t(_D2Ox2KVZiQMk33 zl-QfUwomw_>-efuOFW39N>CJXjz9sjfD3HltLDiO)O9d!{KEI*2tj&pY zVqd5A$Q#bEbi=sNM%qktJ>o!_k}dzJTjtaAY&M!v2`T2@&!KsB51uc`WvgDy^r-r8 z@M`r7_~8w!?_?D!-3p`O+rlm0Uu4yU3%)rQ)r4&Tt}640=EL@E8gyL+dNi)eyzGUC zbzN9tqfywoHRMr92|8=9!%ntHtneYH+ap;)13wx7(TgmTjTi+k?F1C|9~fem-Ma=_ zY%HqY`ZB0imRrzuuQ8mNM9MX-u(<*hSkHT4)Grk0WmMimRj-i$L4h&>vuDA#TED|0Uv z=Daot;nLws8|BX{h7*uqJ?WJXc-`bJtH_To%<4zjlt13KHJ0DAZCTKo4Zz&uH4#d| zEQcqio+ET+~i+B5p9Csky?n;J<3%FSEwvdh3SP^M3##;EWvxO8lA$R`IOYyD{r_2PN^qW#~qI?#|jZ zHe$Z_)1im&fS{1-=KS6>f&n5qBt<}~92NPT7E&;B_1%kbv;)9+LSTFw6>nVK66V?# z)z*A>Xk~?&0oyJaID7mA5D0L0D2d;_m=V5U3;P$`8a4tzH7Mxy8sT_d)mC6;(0pI& zPow(dpLCb?bN@d7lL?oHzkW0(upfKh;=vXm3T<$8_J6dPmpFXtN0a*n3M_am;^D5u zT0biWpggsmD5u|QX9O@t}a<8%hWaUbcnEZB_HQj$W7^*LaTUI@ZI zas~Sq$23r9+Somz6&A%zH>ICPg#G!gko9lva*j0AK3GdaZzZ>dz@eNF=5XgOzhg(X zTxqF2q-S4SX5@xbp}AyP$n1vYro1MW8MAM=`tC(|?Hkny4b+fup%sU@O5g0yW&g+%^M3LQ2`~?0R+Jmf(i@clq@TG!vx8SM%5k?Z<~=>F`CaY z9|!DdCc`Zx23g*;r8%sazqRF7ZJChoUJ$yf8B~nbL%w9AZasoVS1bbaK>)ssR5hP$1%$l=AA<|eOY9>29O4SSEaN!FMKtU#;88=_PR*}BTX=4>&%a6@komoZB> zlva}b4K`>FubV#`*M01j0A_$!BBX(Ta|qbLUYY};C^&U)+9zAbs4vkPD`-O37}pHu zIr(izL@{EjprzVq!2@v&&y@LMVJ2a5^GBGW?(HF0q`iK-ClQ?u?v3*l1P{ENJPrk$ z!A@oC7`BDzV4zG?OUlG=e0x8*DiC=U)Gc7JPI(aQE`g$zt}tI!4EkidZ%iPHZ8BwV zHH8bY!$ExMEfi0)OQ6Y(7O9V4Z@Ia)w#C~^9;!I#UoD+?Y=yyhov-$&C(v~7_ge`^ zX@~Se1WE=C^u3c_YjI>ev`?-`5GAyK)q5>!^W6PW^zIq4_xo4+K|sKE#{)^Y{i_?` zd3A{al*S~9@r3r|PQu`CFU%QtA~p`M|AuaEb0XF5&iNa+Fu_aK#yD0?zXw|%lPv7@ zRW}}c!>fE=Qiy=sl}ye2mM4;Q*w;z-Pda@F#A_cg!!Tgh;B0&nuF?t2SaRp=ygo+n zV$pN$wI4^<^J;!}y^BS|wbvYGIpMZI_*}n{Q9&QCDZn=UtHb{emw7bv>=ukS0?J6W zr2gfqCzY~n3Y+8Yuu$p^G<1eRHV*77{cs299EITapkf;h9=J%`($Ugo{^efZGA?o^ z0JW#G{RRiyh;ote3RB3>N)NtCz;oeR!G*_?ZQLk3cn{TN0E}-;ZI-)q8Fu3GGtgU1 zVYyr=6jcf{6$6j3Y zB|opbiJ=9X9$SF8)TeQUM?PpzwC39C3E!@c*UpajT5yxya)6hc=k#t7ePb%~0axj0UK-$;GZ7ph4xEJK? zq9`}sdp*rFg)%fzu6pCqY4eLpVij;8`moJE>a9n;^{BUYum8$=t0&nU*hME5b&YIZ zXAj(>UZNq$v2+q6C1?xuv>*!n?vCyj0&@YlSX(RjJNwqJD;f-`_pi1@kc-wcyWZB3 zp4?*0F#{M=>OFm`jBi{2o@K186AkOcUp@w(=d7l@Q0>qfQWBgh`$gnvLMt()YDj_^IUcmwcWPz1{1e_}>0CxdLwLIXqA&_ZO>(+rK6m=wlUg``2+V{MXnE zNfmnUvm3@L_NLMtsp~sj!8cVpPp9r#zpQYPH2Mk|B$ug>aq zg_y9g761?avtiW^(*%w1kRl{Xy|fh1Sp_T2_^aY_`!r!At%g{z%Lz5b5?9u6<*~sU zV~G?4B~kiUau6MvSg92(ui`ci$~R6jp{m=%G;^=Y+a$8#ig2k_-}b0Z;rXV2hpfvf!PNFjH>cqhsv@_4 zZ=_UN|C*?{P3>>A3SIxYkFGa)2B%i7OV;f=we2ER>=KDaYTXT1wM!(OR{PFXxl2}3 zl5625Qk+_nwtqdAMsdH8Lt}}2QBZUXOtoP5pC5q6Nf@1Lk4o$4u~3>1ZDP&zYf=(m zCTwY>ec?(99z39~m$tIDu>zw+135t}jVm@moliZWBIa zWl~3{c`jVOwO5^S4~n;{m?~vzpygoqjl{44#5WtZRgr0b`BHHL|I7 znBAGS4`3P~LC>anUUcDPjEkei&PZ`S%aIk(!yz}}ammfZ^xM-@jO*~cKw6#fVJXvA zYp0u#FpPjJP=~Aw_nD}wjwVean8;E#!mQyuih^#h@JQ-@+gxh*;@c3OL}ysrG%PG) zWizFXyzlwUR;;R0*}!r)%$?feTg+BjPO>Tt4{0yTl#(kfI{;oYmJ~{-S^uiQTwF7X zSmM&;&w1{*879srP&EY!32leZ4(r36DEM#7vfzYCXk0&zZ8&~zaLrBd+8z{wHu>y( zE%T{|UHhN|mF{4ECGQ+QDm6G{=C63c)K1k9vz$V5i%b#2yL+(f$N&La_u&Wf0I8vU z(8^!NuGJ1x>aqs|@jNO`1#hI$f9o<=QI_e6sf|1uEL64B^hW)qc9T+QUDfSG| zw2)%0cs@qd!EQJtlHO7}xQY)g`FDu>lnN(!adl~)aJ&V?O#F`JL}eScrKqC>If65g zA{>KUf%V%Y1zodBle7^xnH0CIsv)>!E%;BtGa6gqpg_QYUW5A=va&(c!z^Ge`WsxH zMkLX;row?UZp~{Ol9$Qtwh_GEJptK6A1|^}PjYt4iYJP%=d{W;yk?m`N1o7<&!K|? z48DtPo)WE+zzM>kt=)OG@vO8f5xYsU0pagnUB3@bsu2c&shKFl))d}OtR8wURv022 zS|$#SiHRKRUE(pswO*l`B}8J_&;wG-1$hy3k7_=F@H+s+{DN*-@q(&hP>6?|R)~O} z>uI^)#Kv)CI+Od#V0yi_xW~?5SGJ2WVB{$ zS?NE|R3=L^mmq1JyD5oKeoc31c6K6)!kBo#V1?7V)v1+v1Hg30Nl#o0ICl@S4C1K} z-FZUepmB#ag8#E&Z3z??-X81lrKdu5k+$3!KO?lLH?mo8Fu6E2IC)D=${L~eJOnEd z9EF&6gc)?LctuOil`9^%AA+H|Bh4;cRtOe9-pWt>LD#Qee)cswSi$x6cSS5#2y1m>~n+>OY($Iv=zj> zzp1ccj{zC>3&4dgcOsv0&rI&U{*1{G6Hp#24oc>z7KPi24WTCE*{R& z6acH(lp)2(0Izxs;HpOet(pi}<;nd~Jv0thb+1Vq2ddhKp-N(|F_5Y$Xc*}LRSrL3 zl8*Fn;8X`k1s(OQqn>rtvr^ld2>0|jP)`$Jo+cx;r$aoYC-x-4J9UAZ5~xO?od!Ug z5{QOiodyA$5@?P8c3-<3>r1cGQ7Rw$Y35d6>}q);-&dqEFPgdI8na%dX-&J?JjRG6Vj5JLw* zuB7yB60Fdnn$M)!nF=LjK!OsQkq9HS4-k~n#VCYO9~>ynlpKN&8UzMPAUg~lbT4u0 zp{41=gQ=#008It}>cIP?StEa8@SnuZcOdA`0f0Yog=R9?PaM^6BG6BQ1Lbi*ejXm+ zXF9kK4r+w#6kwlG*q+EchCzKIkUh~&*#-1Tfa{4+?1A~DLG>iKU5pJV8Kx&fvY#cC z4AB!EPz3Oq0?!leJs*i%8inSGZGQmnnFh-fo%Wg+pD!fOCGWtk zSjPA*Q`4dKkZ%|%6@*VX7`r_(KBeR+D|9T??J@0ec@*Zr=^st}2gd)Vku=TJQc=Nb z6CLQo7;LD>3oUNNRLsM0A5KPR9HBf^TCm1YtexUNnG}2o@8q{k=qxE4fhagcM;;g6>IK*_q|SWX;(OB~>H11GIkkxk~PR&f{P zgBaY~NkAnguw1UpI&9p$}_lMS>B*Z==?pSMa=k)qcY@FO6+4QnPBy1uZbs zKsWJ?%o#S$Xd=&hV-(j%dkukxa>3OGSCJb>cZx@&)>cggtkym4OTa`hmb-*>B4n<| zIzqASrngr5t3huI&F66qq;$O7v+j6yCVSFwYO3AowkVaMCB-(p7^aU|U=}2|6UU#m(dl9pLL1!&^8rYDHfPV`Ds}~8Xl*~AdN5jEf*~tf!w{h$j z<%W7$#Lk2TcBi=YFBqwA?OBVkGIew z(a?e=BL%J2Hb2HQR=Oj{Bt!(^ge6y) zFU9iNNH4Vf5us`*7|0WY%OGUis&-C0QeyP*awsZhWm#9I?FEMI&_YXe(1;Z0cx?h- z5U8teaQNh zv!;S@r5E-W-Rjowm1Z4wabA93TOyuy6C5;2M%Nr&TM!TC+SUs&IYo=fbxpM>bQ&Jl zB%fwyv<8D*+QwMVZ`9jn67{vr+0EOZXlIV<3Y87uc zE?VI<#I(Gq#at~wqB#+RmEi#WEXHrkl%nhDgJzzyBeiL^RLmUzbf3e^D`={Y`z5%GwUj}`CQ0uo8xruzP>o{=rhHe8r<1kD3_ z2u(yJh+XRhq1Oxmh||zOK&ZEC@>J~&XKFQWUK?#xk&_NXJ~352j49eb36ltqpCI;( zc5;!`(2s1ee^oKZEZ&S=n0cvB+{NR1gRN1^R$=`6j+m@Qj*gi zV)02vQW^}(oEX{{Bh?emBK`CeAcn({ZA2NED5Ckt1U*lU0BZ=E2*5tlr;fx;{C8|J zWGHZWSjS5+IP7bH!y{mIyM%QRUC;Y5$rGmj^?pEePsvCNdO~egeA2B6$HUD{ zFp=%g`7l+(hP5iVE$cWVo}|RzBkuX3GV4>%fqse-Oeps!((cURc}n22-b`M)w2(jq zQ3A=itC?&6H8Pg<7>;I1BoKE6A#tNXd4vW+l^d~Tbtj8$2DbeKqEz*e5r?2_v6GJr2Y}ip-?Qc=1b=8%Q`<6Gg^yL)8>{5AN@u^Vaf)KaL6IKSyRp@P@#T zwJfZkXn!y|18}+O=XjPfKUF@3{f>?r>^h|1BQcaN-#dCBY}7T)HYPO@e}SDnyj*LF zrBtB~B|`awiKpUa!N{LnR{D&sRxGRk9YyIjKYeTpTxR&6?5BSpn$(zVf^QvLblvKv zaBJrfW2SpZMGpwHc|l$U;pb6x$I6GaC^|%xku1H5O@th$6(eutwVs-$U^8+B2~F)c zzR0(GrrNJ82~dt%)if(al2u=Hi6SX z>|jxaQIc-)SduFblDNtC_s^(c!(8^XR-5op+?J~XAM{GW-2XvWd|Ec~VRkDgpghCD3mU|oJ46D}I zH7i$^@3>9v!QBOj`k}ArkIv6MPVRQ>g8XG?GqQuRv3{6`X`|UTW{O%Su#y#|8J^{m zgo@6`@;cBqIzNx2--z?t$3H>j-{hr$gWPPq?tGT9Or1Q!uTEmM+MZ z%ma(l34_Rtdia7E;MWKf2AVU>TnT#wMYCP93jDu%s8EUJRTqb(o#$L`DIaxhr5LoP zfJ8nHJmfpvPDL{`C3?=E{j*kWFI04Yn z01_q;Js8Sy!qBC{H$Db%V>)DGGCEokSYr#qm_RWBrm+XVn6eK>0U8elUQC8uobH!S z82seQ8xd&5L{P;g_qihknM>L77jeDgNi!Hod9BvDlm%mnmf75-X?JY3&X;U zhpl;5!Y-t|X-Ye%5ZR{R$VDc%M&`!$f@&9n$Gc&>IVyg5e><%hJKj+X+KvZBZ!GgB zS8WmdDMK+z)`t1}y~J^YGN4AdTFGk3^BlAxYyjco`5#b!wKws4$8>a?MgIbRjoj7_i`L1VaCkP9N30+K(%HBUjqK)VcVbz{(6YsOG_ zC%&7D8t8aE7;&JR-W|eAaV7JvMm#?6qfl$)%zjupyVUt;UP0t&PdI4e2VsX(=>bk* zRUfZ!*j*BjI(>~d_o{g0Y+W%J8{A&7-9ou)OGX;OZke9;oGasX%#EkvN4;{t6^hne zt+*cG|MgP7k+t<3{5URjBV?psKd02XdaYIWwgj!KF$>Teps0_VnO0p>g53(}hHVvi zd(N#3xuk_HY%+c}z2CdnwcPyKS`a!72<=fg*!dcQ4$z{Yx8YB$k~U?lCEB|&Kv$81 zfj5{6@|ug6cz*|FV+L<0U(SD$f|0CfRL!dKX|l@*;jG-#ZZ3d=#qrvl|fs_26VO^|?KmYTPcX%>LY@?_vc;BWmSP5F(Y&6ZoXbNLK zpKm=o(o>$&s0Ao%$=6Aaig` z6HiG`<#3~uboPGt=cB1;b2`tv=#GR=c<`Z&I9DbE#tCwuy}uasDDebmQ8uy!f+4Nn z%N^ungk8Df84CdHVR=+OOTzz|PUbje`A{0n$ z!l?5%@h;90s;%F-NwitKpPW&6ulxPV$HUSYhJIYY3Gu2{aMf-(AGkCFcJ1PmH2hIe zYq=$%9z>cqv5Jm(M4uSZseq2n>(~*no}$D|%N3= zSk~hp^XhpV!UfcI&+Pncco5Dpz1)Sl+t07XtY#h;RH?-xz2kjj0Vk^f=0^$Q zWfj+7!sNNP7dk{juld=cTra7keqrxg{qm5SM?_0|$9ZCw0hTD|=2TMxosLJoVOYvS zDg({|Aq%laUM}F1IHGXLQ65=zx;`}(1`k@bE^Aeh?D%IbF-6yDj6EizXUQn!?}|pr&lJe zhrw&zwAy!c^u^E%BVF^}mQND9Nxb312(peXznc55FSz$&zXz)rxvDpntSUA9E+!lfg$h?`jTW~QAA@%QL-t_03j?~xMe?

t#3P zhbsFhu=dUB!fPeIo!xde zk8RikQ?;`}mHj5%yI5>u#YN`x6cH=Zn;p5@mU;2Iq+Pfp2x3Ej2`S%&@w#Tl+u=>p z@Bnk;^2%85&&~C&-RVE+Se+kzy?Z!O4p_PUOuAN)`a*MQ+66ReUhs4{wQxO zn@Pjcu=FZ_D02k;WxaDLg{l*YWf7?>gEP1owQ(u3`A$zXk?f5(K$Dqc>uw5tjdRxG zg%U`xWiPDF6isU4z@G5#o9TH)_(9s%DrjPW(ybjV+tt;f+$jddt+EkrzcSM^4pm;E zMf5Cg<;mum+&z!_7~$WrPr{&7w0K)BfWympT@7T~w4OERH|ykf6TIE1 z_#1oiX`}mk9IVA^^K#L#Ci>`Ki=Qf6K5|jjY>=hbWdK%$5y9hgKy|`O`;(lF9wt58 zY{Oky&}#SWh>+Ty-g*CyMRbnJiXfK*1dU|nlpp5QzV?NoRG|vzX9c&@SWJdKog?HA zVMUhvg4xFobohY(Z%^8iM6vN@_>JNgZ_^L_3*Na;BXkE~BPmnGL#1qZ-yi&|Q`7VP z>cVbB2`$EvDd6rQQ+yzBuKEUYb#9p7?&1G%^1n07yX60KsQ0?tJQKhi;GqBl6pSBT zZYk$(_n0xX-?B4F!`ZZIfg(9+p;SNTT~4(FxG3>~sigOWDH-9pwAV`ctsyPQq7*95!-rk{|gvlc5XSi;kd=z$k+ zo(E^RE927=%m7YpmS#Jb0?SN&@;W$iZS0)eL(Sw*v-d}M9NB?bdN$VV`Zd9gW6zb2 z>O@Vx-l_C1;9q)kH>D@wuPEQzwrf1CJU{UAHf29_5JG>z=fkzDOVU|5I9qgGHV9{q z4wBBptybB-QSX6{h^^Pp*?~M3Hm0_Tp8n9fp6C!T(4t7m6O#T>tyNs3g#So(qo~r* zdU;tYIIh<9k)APEFmp#j#YmvL$BU+)6an1T4F@4;D3Oli6%Qb1=OzFt3If7n6!wxA ze4bJZtEDu!!tu<#`KF+&Rg^A%m+3@W?NgVsk*i52kMq7JgC$Wt9 zNk%oQF{)zU?`$WUL2CuyNb1nlxg_s1;suyy5Kz@x3}ddUA@4kEqc;KJQuFv1Y=c(T?Vq&u~b#asGUS$N;|vEd?N7JDo zw}`}TqSvV(8_xB0d%Y%bVm27&CFBZwF>h^QSPwgPzGnY`#WC*ej%;{bVgv1(eHL-= zv@QPgXd=BL9-Mhbm}UjhR7lNAd}_~-*2QQ(cZt9=IJwI62lCJOLc->m9}BKKb0sAj)>$z8laDGh$Y$I2Muci&cWizY4%$c%YA zKH${!3i`a=ADIX$GskVU$LYRMs*TbjcdpXVQTtJQfvB*zzt|tyK$$B?eTkMc87{p}6nFaaBS1(ah7tPpgz?w2#%B@?9)f1D= zDd3%L$qO^L?cI!0&g8UfNKR$~{KS@kU<`A$#9-x+ZY!)=gr(BCkjxNjV+%6su(bUL#A+7q55d&4>wQMveO~9z2and+-evz5>Tci*q6Wr^mM2a{8<^7d5?_}1`HMExUJmvsvi3=jo^(z>~=YFM< zo8mk99}L}i+@6Ci&x}`|nuzZLuj_)9kDI|}oIf2FEA;bPV#Z7i9pk+9h=y|S4erdy zsGQ6fVvdAOXGNg+?aX)r|8yFcI$&bG6ZEhQz*?U1B8H?cnkhHZ*`O!V;z3Ee2)VN^rrS6_N-FKf zX1{<@?oW1WVlo=`>)1W+*&Xlnq(4eDaW{xknu26%k@Tp;fN5eo9VY?1s`f65zO9bI zCpn^LKMo>T=x3^|8ef+cQ-nuuvLK5Gnd)oC&ictNr4$%OTftN2)k~N~S9U?55q@A1 z3CoXnvHzX?h8DAGJvFdlt*KZsif`VoL5B?M<=wu}lG(uoJ1mA z^7Szi-M#@b7RSCpno=GO)rE(wpzp~*-tg(ep|bUsP&qB_nI>B{QnK1;Y`7w|J=ePL zOl)tA=siE?5z+@@_bTD6R`(1w*_S$;Au~F8&DrK14KXb|&@P?y$Kmu@*2Do3G_xN(pE6e z2Z4(P{6C(kgVBEye(wb%YoW-wr{CHFOcR+@-iDhkVIK%F8~TIp#sL~o9Eub0S5j4OG5(g1z>fH@RpQfT-!4N-`}^G$K(w}@0qw}7`RE+At7P4(BM#f81(JY zUIt!BIb>n`hF#iJm}NrMe(($6OwU zny_(8^)ygoZo}~>hqO3r0mZQk_QL#pnn7WDiy~5RfIU8wse|jaSelN%HK>h*h<47n z8FiX{L#@k^{8N$4m>h+~{o&y%urW8I{-T?~L5ys#Kd`g98NrxxOB^0duo&##Eu-{*N_GQgL_&j|OWV2f)9P=&fb0gc9L%JbbFkZM~1DKcay}ZVZ%HlL_t29z4L02ShE4{;Rh_G=E`V2>l6N3UpZzYyb>1P^kz?hz}DLN|V zwNd*tnlcuAP$Pwmj+;uOAQzRYwX2=Ps;Z#mY9DYsdnsc&6S?~A)4?^f&t@gR3&U7` zCu%QPPIS(@;lNw{CL+#>vQHPq=9COYJ{kGp>u*n-v7xzr!C5#*9~GSTpZzfvUN+jw zg7> zB@)n3U0vp8V3FyYbG!EyUR{KQVh3bEC8P_A=%@{&FVY6+<(*RxwRb3HYnAH@wi?xS zJJc~wT^A^BP7d%~kA1pPkq5440dt!O@C4|aIZzQe4qY=uAWRV9CAaMn`r&94YPy3+ zFj&i@QWHmp=V83Y`{;_*kCrBx?d@xr$|w8c-V(XcxG93@z(MOR{wH~_ktjr)j*Uj! z*(3}@lrW>_grB6^F&ghcRd_mpXr)B;`dsbPooj_*v8r5ArE(mmHG%JsDR9M*MpxCJ zm<;*~(fhW7mlNqjW@(9R`K`HP$}L^2dnb8fmPt^M77~~lp#sG%G!(ENiL=79AUgAr z+6YW2UZrbj+SwVsDs$uJA|U{s^;mF+?)3@6D%hjThtNe0x}xe0K?GQ<)V(eYVgKJF zj^HwOH)rt{CQVi_R^t^7%oP6ImnD`ycJ zKi^gat@TwdK*GWf8z?*Qy-gZlOjI;z$7Y*I_F?(4yfwn_KRaJ*F# z0QP7|(r`ekD4Q(UZ8R1PKz$=R&z*V5C*wrulPsR>$}#GHK8Y!9MDQ&r7VJh$u=fPXoQPG0B*UP%2Lv24!*Pz)oWznCRdZ*W9vF7t6 zpFW$6?tx%FSgc4O3jw+-Sm)FkgXSqQsWAK4$n+z5LHV&`wJ(*S7oKNxVxg^s(;^!< zNvcN!jX6u(saZv?%t-u0DidIYL9e#HFFy^~#}?WPDTi6QN}u%|c>11bpG{|zVRWIM z;TNv981I1C)n5JsBsp08!2iupfvVS?WG642sBE($yAlX6_6dA&Of1$5^4*KdnIbN} zwU-!-dD^i8uH7MH8v-l-rrSDePIX5}bvlg5iSv@JX}xy1C2%BQX~9I}+{`F9Sy>-QYPalJC{o*`<#BWv-pgQ&-n0?WGm)izCdzN6oO0o54uj4KEKXqAqb)E=~% zFfvAHBP_9Od(dEL*QjNn^jIyusCGbo#SYjAP)1EvI#TVW$RRc-IQpJpr+T0c&0 zy(@|Pu;NfTNI>(7kilnYNK2O?Q+am8Y5*`Dj;^9>r2#Y>&dm*OTNF~TssgeHY``V; zD1!77L|Yje>M0yb4JYyw9a3d5tcaF#bQFgh*{ zN5nSh`4TFfxy-6n7|k}uJ2pgnbyyXKWG@?L1iA+6a9|C)7L7bcMu6!+6?MkmiIWTr z1rt4x_J~A_*iBRi>y#qRSKf7%NkJq%f@XZ+G}XpK#r@$pW( zRdpgJ4---PPPI0xoO^|hn&p%Jc|x1x5S)T8GBj#><0(6kj=MAs3>D%M&K0aR^(nU} zB`IG6ZURCX`);{WBI_52MK(R92g16?xZG4=&}zCi=_c9Ysb2bPKATE6Ge$t$b=%t- zilxYVR8JqYdGK0n4TRIfcWNAdDco|8wJv*Ku4f+3wB>J1>-K;-ze!+B zWod85%r!5^8yY75;b3o?cdlRq=+0`-HkqN3O9IFvh5*uuhZZRMj(9UU?FZM+-)*S| z)&w?C{p`Uri-OS(F%l9Oec7j$(c_vf3$JDS7TrqK_wjyP?cTVpyCh-cJGFXd2d#y( z;|zM>ahPF)lJU@7)r!OU>)?(}nTT#SwM>;lCgq_^bD)gX^>weQ5G157+}ZW564)Y+TrtxLZmIXoxJtbth-#b&gn05rh`hMi+f9Z*?ADJ{{>K!^*Rgjy7@LjCB&C@fX8yN5O)l**L=rh5=+4P`=$nP@#P2aV zw$r7GVQ3RCcN<)T@buQbQ}D_6eGw91h-Lk;*^t2{As|+ztdhUxNQWbf(QXQv|d`SD$V9EqIPq(&g zOXayQ1Iw;OW>`=+GvhfcB2VOHHUEr64}Ez?nkHoSK}yWuf?K@0p2($ebe^&`m}!r} z$XGP=U&Rv)2~|s$6Jbq|f5Q#3hF<4GYbe9VE9wrWFhcMAv$>>_%a3%zElk;9r&c2l7PzW7aem(L49Q9rEH%uJ`D|VfD95};=tpz%zhAI_q@>m z{l=GPJ7AE@THP1;+R$zHBhl+JubyOt4=xxE^!pWR8Lpnc_UR?N)utfS4#VC5Cn_Cw zsM1r^6Zs6)S@)MK45+#uzaa5E>>ilT5gk=+l68=|1 z)9~Im;p(4A8u3&4)KRN39Cdk@#I;_40GXB``Ek`F9u0fr4B8#K9$U~Bqx;~$w zs_^Z`OiCX6q;Sajw6Uz422w%vub;r5zO$oV+Q$WMVlUQl#`<~MZ2{sslFl1nYL-qt z*SaiN*B46AL)C|g4%$%M z9)dKaF{w7XFpr&22xR%B{U>jn0cyYX%J$V1$hIg>1Qc8^`=Vfc?HMks!5s2LJjRPH zhpHS@T{J)Sbq`_fgH0d|c|0Yh^~RAdeUu3z&LxT?NX7E?-o{S$EmNE?Kzni^;krd> z@uSCZwT4P1IwQUxW3FpA?nHabK6N)09x+Y#+F@)@%I4P@42u-XYngRMa+Rmb@F$^p zlGB?nUDETd^IeT`I;PsD)Rp8CCZX&FW&y1WePr#`l>!$?Tgp2BnSoW$K@^(rhb5+- zMjbkzmS!)aAK8j!H!*%=fe2WXA}lbIVD*N-SfP4%owhsT$^n4tt)`rOS9zV{=Cb9^ zcTHs`W_=8}aZ|Bd9+i=}f~M+%6jJz{R?6*zEBiW>eJ6dpahS@z*4nybE&%5M1Yse7 z{Jg10#nnX^4f7Qo)|#u?;I7jOP%bH<_)&D4&|;K1FBLZKmoiQN&YG`FG*N>L?dPx; z1KH;7ym}(Lh-Ro4gDL+#kAe8Wf;Ym}6WA}F&Pc(bT0;v`5!E)nobv)N-0HEp1}^R? z(^c0mTLei(Ym@H)$`jW-V=?D|T~s19hFLl$u zQIW-W!jg+OPn#SXYcjR|xCA?Mu9g#YtH#lk4P!CUZ-EB6y*ZoNuAzw{-2gG}?2%iK znhilbcVGZ8JodV23_s^9v3qN%_q#0>MN_b_)+WocD0P_!tRaGaT_&qx)*PBH&h(TE zCdr4>!Wr`pHlN3x%FV_%p!8)Nun2U^sw^xcBRc5t>|~zEThh4=H=%~M2C#j?N6GB? zBO9Sy(X=^xBV3CQ18~DL#2dsovzr9tT|Nl#BTM=I%JYOb9_t|1W2&`#f-mTlagf4H zbc}gU(e8l=rpZ1~AV{zjPOMF^FAzm3^*~kNmDCTGTlyovjf?+STT5JEuZx|Duv_Vr1 zN&_q`bCJERd0x#nyY7ymaOAEo_ySd2L&X|Uz7Y*;NTT}YvPt@mTDDDHJ(5|6T4b$A z$5FSF58pvc@CgBUxz@_U3{<`B+bg`$<^WO+`A4UxaCdcAz5|~wTfeqzIVtYM$oe@% zq`#Z&aEd|aN?stvFo_x>L`GU6Vi6QAJOeF(DD9o4myxFDib;O^GO`Qq72|`1Qq?;o zPCxU&A;s!wfGS_(TMmy;^6xnq9)q2qZcK~C?JR2i#xBl;=Z=E-7GHcq2n5@Ox@IQK zi>FcvES{D2x2_l0wrFL)3?K4e)Y9*c=k4L(szSD>N8M>kZS+^v*^&4z8aI2W0YV(v zP)klI7_VKApUU{^lIP^d_*Mpe zsr~&7{d(o*K8RX&?AF)zUQeGD!RG*ru3Z`3Ki|G)wpGc|VaLK=TX6Bdhzu5?p}p7W zObF%QB{a|*pMwkOIT;Fxf8Nr~tK zXbVDsynkTd<||u))2d_m+PQPI7*{kbGa->0EE6|fC2aD4t!HS`$sS^H7|T0g6X1XQ zLW6sq8--ss;>Cx5tioysRGPN3n}&mp=WBcA^{IJZfo=5&_Z?P9J)?3%4)wxxH^iAe=dh?54J4g zY18A>1micq+a&k=T>Z0n8(I@qGa4Eqzu8q>FWOP#v0*$@(&R_q_?hi*SXVn#TKbkTqxS6i%GHBb*Q^Bw|L136hbU=7e+*+gTQi^chee8OF?54n z72&VIlLOY5R%&mHEh_#JOS~x~y>*%JPV`>x5BfMv2;oRy_oL4RiqGaBr}{6^VVSlD zVoQf2C&<404!UDL%Y-wUih?|0OcQ|n)=_D6&)d~#?)C#aqm{amFq0wrEI+3mN=Di( z6j*DokFbI{IR8{75x8sEC|$%xp$1cNvp-#%B!a^a!z0<^30MJQ{qjPOo9hNwjl^;? zqoLvKB&SAN(;ABlJL{k3ii4;G?j~hPPefSZSOGylNH@1{`-0($%tmA1p$>~+UK@bi z=(DfJ4IJl{l**ga_|J6fyD$xcQn?yx^bA=IEYC5g$mMOWp21E}zmLDOr~5ni``Ga|*!(opeCr>o?X6H0nJp zoJ9U&R@TAy3Qn)gqSe!syc8=7^gNBTsNWK45OO-=hV0qY@X z2BqihOsKX9W9z!%<8Ckdqx;xUJ~yf`y^az(W5LQqo%b_5 zG2?xo>^FY(O57`~$+vWDP0lYz6Kn4$eo#rWo8$kvt;F;|X^=qAv%MVvRU5H;)O&mh zjkl@2)vJbeHQ^1XmVES>S~=Ube^>;}tRtgKmW2EW*&aYE5%CDy)jn*%Yvg#-kWxEi zq{MM{H6Jg(Zj_*u6>twC!Aa5-aF`@n`Qm%0SI;tfgvG$OdDs@Lx7XL#>+mT~F{UU9 zri4f{(fviz&7g`kYizb+Cq5mc)4te)pbEXgl(t@C+un$|S$lY$mwc`1_AE z^zhbuX&~z7_OSaf*jAynHu(#puDu7kOhc#sJ~RkBC1JJB3v#UI@`PiMf)UM)DD%xVHbqdYnWWGkE#Qj}r>`f7#x1g8RU=iJSc?N98@HWIaHN_4 zC*)j9#<}uWV+o|VegWfO$9>0R+Ik{3OSl9A zg43y?q2OZ=(^Lh6AKux=f37_t85+se5~ySNXrSo3G5*z@WurV@zW$(UXz$;ShCyVn z{eQl7@x*x;xsrQV4Xnqlv>e(Ilq2M(=cnWv_`NW;utD9!^&yGnVpA zO#=>_nizWFu~nbOU=4%aZtW=|vIyROhb3vF`oa0^68{;$Z3(0K{|wjH&&`;(KRO7c%aF-xC6>|FN;xZjViXfios?XVanEz{DbDMtCm3buC7t(^m+ zv?}t>J1)VzUr*aksz$$;r=PMr>e1110iV6CRpd-Vllmo z-B7W*sR@HwulcqvY<<@oFP(9R-%q$am+vQ+tY&$$ zb#-D(sb;yV3)oLr+~>Xlx1C^uU30>Nk*L|H!_SH7*p)jaOZ$RMg^4pIxv^&3C6!WF z&Qwn#2^|E4p?gGZk%}#4r$d1%y1bL+Tcsfeic6Mp?@Qp(#%w6KS?vbx^W0rz2her2 zwHDZioeQlIc%(&+QCqJ^8e_lE2I|cVBw}y820*vwT)}`f9~8u0fnB> z9bGaY2P=Nx&I{zE{QBN|P?Z*Fb?Akm*EC8t5)m7_OZ?1Vuia+zRJ}BWgxZt)G)sQD z%}tsk{lVP)s5FQpp%d;rK4GDp|-;GV0UDiLFqek zM8O)urhWq@l?7io8tx>k4|f?&MSiz`w)s+O?nc9<6LHhBGy^NQ-x4)LYzAcooa-AY((|Hf>wpJ4_)b$n#mKg)eS+s@FYX zwob%E4mH&+_IdS+1DpEV+k}TS$5#v5t)72!tBtL|_uk@4ZCxjbE47xpy5?6BwXC{8 zn4jx6KIeYPBURS;LAWQF!er?}+jI*K{)=c$HMQCKk$+lv?ByJv=ZVHZP^WFH39jGB zI%umOaPHPAJbVpSw@QctjVs2V;m927FNCwd!E*hFUiA7J$nb#GEHlIu9sDb)I_xw$ zF}tE02@ILE9%+&dE$0wh1MR?JL<%tv-KnA?DlRCx!I1b6Hb|Ul)<$Kf0V><)$RzVuERhK zx?tU2HXmhMz4i_|LPB9-F&HRGDh2K+T-ACu!Zd!FsE#Yc7w5#^ccS^RfB~^&lV$-j z+z1)W65_WJ+pxF1ii8L zP5K)8bDr<)e6;T4ZqN=*sXYso9dDOrQ=@*PBoE5Ykqp>Cr(Y3p*g#3>b~_ncI$PM@ zRG*!bVi=8TJIzx$)%vPpI8y&23farHpsrFRWRvf^)Kxyj2XIkNBS4aAs{8- zF{tiO7<1kJfQ}OiW&rw3J%q?%OE7Kacjj}Qa9jJ{)XVtyv!;xF5W`l_s4t&z%*(?P zw#+CkrHsDC5t!PShd_F#%y+dInA(2B8?oQuUgW|k(Bbt`*^`K8V(%}@V%hN?tTP^z z{)A5VmDZQM6igB1^?C;DkX0KC$vRwU?gYO%m;$A?LH3}J zq4w*pbS68B-71mM(1*TXY#KmVq6tlS3{4oUv3YQ#`Seft{rHsg7Ssy3W_dTFT}hiH zvTs306Gzq2sn@zvf)3c8RF>kvN-)}=Q9Ebze^)-j$sf^{O?WY+Obun{9>IoJ&FjSo z@~U!cqx62@vz1)^)8nGQG__O8}?{=@K zr?ACIC)*Zyh8(S{7D!CYh#N;_s1Ry2?-97|*gzy{?NqPkquS2&JuG<@e6v;PxKq%c z-N`td9NQI%B+9^fj!QI>;Y4^3E0}=iNI*)lTf;#s{SwtsW-9}$$)W|`Y{&*|g2}xl znKY1`(!|)H=DG|(STE>dm#o-blPR66QwE7rs9u!%9$?d=V zYY!MYehyE07PFChI*0R-`0i6$sq{+3iJ^>0GG-h5hHwD>_CpftTR8z zYd56w(=F$Sf@zb%E+$;#KYdnqpW0?Vbj)*I$t^OX>6xsQFQqh^qUl(c`qCxdfy-_5 z*~(Q8(q!Xw%#9k-Xyf$EMegKLPl4nVJcbTA7HU6w0CAgj6j1GB~HdEB{ykdsp;KoF;&V4?<=0nk_!EsI-in*!$py z&RGc7tJEqn!C`8YOiOcri>TQ(JNla0EJ~}NOs^*P-lk=u%}6tMb>^a&S;_R7@2}B_ zM7Fx#LV^t>dNTtE*>u7Tf+&qV_EW4!TGWK&B#n^goR3Yj1D>-4y)sJ)rKWeD3oE4B zChn-F_Pu;Glbv`N013$Ej(Od*&Q>>#H#L2Y>l^NmTfVnHbyn$Og8=TltzCPv%`DA5 z_j^D^z~_pfCF8G~&&Qf(bHMj!gTgOI#P9pbv|#U7#pif~K_4Ig_odM4}2`&DzZS#ar0F|{ddi*AoWW1xgC)=}E1 zfJW5T+K^0Tfi8zRI*-B|7kSPk;tCqaw`+(yN+z^#evvvy={OfHT3lU(Y0~eLw^*Rj z0>s^BMfHGE@cGW& zw@4;F1Bq9xx|v{sg z4g=c?(~X*SieDd`X5^PLvy4lAGCIlg`|@nm+}}DW!OCTmmJJixF*@VvR8VT^dp{Hh zSYx!bs4+O9Ffd>JYbwRG?I=o_vwvvG$IIYa(S6;^9MO?=Pj4ubEePNDJE#O@&Leaf zICv1MhXuA6W?vm&#_8V%^1cy_!0G*rLI^sJ%CNLg`jVfGl!I5b_N^<3c@sd?S~vuG zgvgj25$n+wu=R^efc2h?RXfn5fWI?Pk-i|P37|MSOkVXaXv`(F3<=s2=FF==(!nvq8OuDZF|QPto+6t<|k?zoZ^L5eo9I?od1+&svmn6UkY zn;|J&9^zb5QXIkbb@#Ub$qCzb3hAC!31U;l);O0e-G48rG_HA7TV;S^*$@-%3B$^N zqXjMKBrw?7Uhzkch}@P|nYLXZDuyubTK^c(U=4+9_0v@17dpS~H-(gUF=mvmk8_cYB=mxDorc7y0-COKd=PF~F9eBKAY- z=J2bYt&j-*=js8@k^@`h!Io&W4dO-IbGM-308l8RUDD|sck>$jQK%2C1~!TKH)knv z!e&J7%;j}FrEt?I_Kv)8_Y#&~hcWEKk*%cp5Oti)Cp11nIoN&~+dsDTDQLt$aCUSHY>pD#kTG}0xG7zFiAnRzw&t!u&$%C5Z?6Ov zY=j2`kwDz&Tlo9k0!GUCZ6 z{(Pi2sNIMnX2@i8g4}y{DvdHL*2jPUUNSUSjAnvap;To`I718)pgmRWU%QXVr#?7V zg+Yq`{D*Aod?4{ke_+tqg7E|R59rr(0*Cw&*QNI8lN5{;rp-$pP(!ewQyzf0D=TB4jqr}46(q=n_v zPM5j6RS@EBhSX+)A@;8SZB{gN#*-H&|9)@_N4fD5xy@@&_A}q+MeOL-?z1OH01JQi zF`D{>t8#}tM6{=)lfsas>f3pWwo3V{C)IWe5){1)tE&!kr*H_C!-9!3AE7BU9RMTq zwJPD*`f*Gd8orAX^@nF+;ete!lo{NEJHsR}iPs{@Bps6hpKwwL4K+Fhl_}D^2}wYY z{T-M}176pnKlybVI4J8n1XcFkh=0?P`<#*yl3HC4x{E5DzwR3T+K|{0#zztB>xl1c z>MLV=_YZh}s*aYvkrT#1yU}~YJe7EPk%z4%^p8YZGBzPNNO&)h zM*S(y^a0LRfE&@(2qN4`WxnouU7vq2LUa87>dDaT>~6@QKYRN1-Vu!tCfREjp4JIYo|UuPU_vh@Of8+kpr|5hA>h=C9z0ndUf5zE%fMPN*NT>Raug?RMG9NXq zl>|SG z3xX0@d*|}MBdQP}WEF$6U2u>$Z0E%u;FiWiD_~F$8@9P)^lpmNSW3w$sH>&n?NmGY zR16)vlARxs1`;u`e*3?ByOF~ReuK=Nwd^vme8+obTix0Udh@JY8LL&CtWzFe?wy+m z9>T%67}?p!A%in75`GPLr6Ql?eDR^(I3zR7%Wyjy`#?6}zJlwp@e3s80bPz4z&cYZFYj)cGh*e*HjB4_ zW!IHq_8+T;;|hP_)jq7Bw&$(8JSV7M(QPwspw=z^0h@Z^#)0b4e)*UG3jAixIDBs> zekgZ69h=#)Q?QAxh91H}i*^nXR{VO=(u$*(S;f7{Da;@wR9BDu!TF>UAOVpsxs0N^ z`I*1hCTBT$e}g+{*)2TwX>R`(VcTdORBQn09EW#TcB=^WPXFQCLk>g2!JS)~K}iXK zuHyHknKpvI=^cq)UGur4Y~p~bTg_;5>NSMoL>bu8NCyMQ8;($B->NsgGuWpm%G)bD zI`PKNuU)%-2xzs@0t)W=QYg|-7@7iPN->lJ`CgHl*$e@i7~t<3;J)22;b+lJZ4x`XX>A@h zCCA{h#;DOFR1Vl;?UTkTGDF049y2R8El!Wf-MaZ^l`6)92mZp#_S?)A@_3hyVj4No zP1>P=dUwME)mGl$IS}NZ;((Fo+&#pPI5fUg*D^y$pNAc!NQ>+aG8@la*)Ia(@?D;O zo-Z@w_FCD={RNvYz6-(_#V^Wp1(j1XrJ1$!$s3~v9xQ1Kn+35oz;hCf=3M0kI!N+1H7TI48T1Tg3G)mcjheeh}-K$j{L2B?R$aR8sY1B^qb~RWGU4Y3wMW()TUU?y8x{bYfSL}$GJE|SybAJ zXn`h&uk8jG3LYU#j0ALe;OC@^)|YZ9(@K8VNqLVdUt9Q7*2AdGmMd+m{p_(pKNxP7 zw$$DgB}vNwQ;~yczjKM;6_ToXt~3G+gd?S@`foci)mBeEuz{N-8~Dt00r zW%_E@SN&$pZ-0_vhLu)Md`Uh%cmJzO)~i}^@TTV0H&^Yw!$L(Zp|&-pg%+~0xpY8` zA>Q^}+mneyn^=)(?+zAsqG!O}`3{PWnq*e2ij3 z4V08>eu%Y`C%PAv)5V9Wk5B-&KKK>ijVb|-JZ6#%c&@fId+ti jIzB!DITb0TU?TO?Pe1+iKjFUt00960eU>K-j+@iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYMcN;ghAbP%zeg%f?du*pwMN+a7o83A0D3+aQ+7BJcPM2v6dj4?Jse$5dZD;&iDtT&KDHCxODE&?6BVQ&SQ5LS-5p-j=%<=UC6lbGr|99q zc-+4lzFtX6E>=_Ya6BHrpuKg~s?v*H)KU?VD9(%CYBkjdsN#$;6)stSWqXjCr8)R2 z9jE&{xT4>?PnUZ;xT>0;jifYH zZtAGppwr`FUOwSW30@RLyub`E`ewGK;uih)HtQS3Q|^#wKiz@G^IcC`*Z z6mHjnsoK}WS+*p3U94X|UtznRn(9$+!99s{T=n67x5^DY_B4q4WSdS|&{)??$aW`f zZL_FnaY6QO#?RhbH}v1%rqg*ZTDpQQy67UL{(8(7f=KytLD}10=wXE|dWoucdh5r1 zU$O=K9<0}&*Q0z?xAmt#Igy#5RYSD{*+Sow*#*ipMr2AN&2r>+!5C_(zp5n(KJFm&%F0XW&XNO%^low( z&GZV|<;GGHB}WBca5?Jo-f6`aI~Qbi#YGN0sDrwb1XssS(%z6A{rmes8knS$$3_Gw zF=YxdA1?$a)dfj& z;@h}yO=X#JrT$%fIa@`;g3p33W~-)ZNd=b$zD$=m|HqP&%bw}{r=p9M2$tZdnlMFK zI-^o@*3;rYOoeZtd}pmdCPCi-+xISthMQWg#attx;i@WV24``{OI4Hc!*r7FB_c~F zf2=DbWToyOJHNL6r$_&HiHn-Z52yg#sQ)|I-`_vjZ|VOI_8xwDr~msDe}?FY%yC^P zWVRj6x#*`Y>COT?o;fv?VoQo1GMRFk8y22633M>e^#DeNO7A~S0L6s)w zIigH)wBofuij-A>6_M%A&Jdj%EDV%Z=m}?Yx~PRYPCLV4`~}9@BVY`NNRatE=@~~# z5HixYzy<+kINa%eo$erHd5S4gqSmCesWk;1GN8S(#=xR83itt#RKhCdna z9ejm`D7Uz+IFcl*1%c@yEcIeuJd()hJ@02t59CVvFhJ0h2fF*Ot4&4I}FO|@I*%S@_3|nO&8Diz;+1dNOc<

_E0ZLmW)_or`{+jTZ?wW+-m$}iiY_~RbDm4USM>!R!49lGxA zZ-uUtZP9iA^Fmh;=C?xChkdA8S(QOFn+AXOH{s8(vGqGv|8nz;_x8W)n=mmdo;CA* zxZN?$h)_pVOwov_Y}A~Tk(5PH<(U?vn{JaY(hBTy`p*Sfb&8VT!$GMe z&nc_lzN&%^K)~dDOD~CZ>MR5%KdDa&FoJ?$!zxOR)$Q?r5rBEUs)%MZb20~Q2W6W$ z4aYc00(M0Y$B1R(f}BG)1m{GYt6S3_y+FS@-Pvhr=C->b9>!Qm4Sc0#6#Z>5vI{WEaDGW87lXfhe7GcH29*{> z`OgW$75!H5x`Lvhx*$#)g#I?r2qGQu%`kr*rb3!Ew+LxECM>Tw^*fx{J#MVDu%VoB z@f^=cYkLGK^he4T!HSy|akc+;er{o%S6D62&zp5oSS*NwioQdrv|%K=c#1kZ__>d> z5`0E}t<~Ido&g%_#lD;(WQZJ5ns%M=f@Z7!FZ_W3^^53C0_n!c{x7O!=Zpi&P zaU7oz2X>rG<>$_J+^mWG)T{QvRgyz}K-7|;)4x33b3SRi@Fl7mFaSnTTorr;K7B(? zSutXqC3>k}8c`bQ{}mkPNGfoNS|F?d?3@<`ztSJ9(~blni6+aJ`jDoRy+7)?iix$t zq~l~9*V#`WM^zqnt2|8iqbdj8DhKJRDeAlwpQ)DoCb^70Lv`ikLNVDkh|sdg7pxkgx!LWx(A}e_7B=!Vs=1@IbI$Sdn*v zyPtfxPl)*)B&Zx+=_#|rGNrP05O7tsjzHrGoU~ErHiR7FoBD`(&rAn{b!58l;_WKZ zHyr*oXZYrGyzwk=ah`8zkLPpBs9J>ss(60b^BEyQeyXV;`BAOq1lt#7i{l03{Kos56x0T-Q_A-TlF34(%23?%--Qw02+IHzMn{;GoShM6D8xjg0=ilf zX5d@0sUb9H%L6b#OHJ%=;%5Rff!3U^h+3eKVt7cc<8GMii4ScAdVY8I@6LXACGO6C z_w0u()R8~RHxh#2)NM-%qJse;7zNF+oTACcClC9UJXl`tjy&9vhwe(;k%tdV9>%{F z^6)Sq4-eD*JMwTx9=a=WM;?CnhW@9nh8Tyq;W>}IOxj1~I zeabnJpvI5lFr)0*NrG`WF zrdkM`6S-5Hf9OFBhv>9s$bRu@5@ux1^*T2tTp-QX%y4#rE-6MB{SQ7vOU^HPf;@*q zbX?*E(K}L0_)FA=f3iL7AVhTyvk+lg3C9b_B`HkchQbTTv9*^@OzdSRDi*GN+pzgM z?TpL}?Xcki9ghs=Xt;~!t2pdbm&_^44fAJvZg?($7#f0Nt{eF5m%7XmSQ}_+4Mc80 za@{!A^r%mwJ`j>9!-N9ogsx~&ASG7Dr2>m=3Gu-FfE-5~jVPEnW;vp4PE310LKhD$ zc!_8;nl*j+qf0a*?jpvU+tUm;AU~;F&4$1H`4=1GY)_&&;FK+pVYeOg-LTyn7wBf| zs~Pm`_BIDA2C>*{duZ=@*IL`Q-`VKl$JW+wsC8`n)q+86<{KJAUl)4cO+Fl=Gei3* zQ$$?iBJChakg>#(V;Q&6LPjmkPS{x@UpPgB{jsJcwm*;@-iTs~-kd&1e2%dBWQew0 z^18_N_G*Bzs&4Nh=na%X1Ny%ia_lZeltGL;1WVznDhz_-vMPwDRg^7=)WkJ{Zs42- zcPska-|gxq2xm4s4I!0ZazPg4?G)Y9-0=DR9Be$2sOtu+uPdHk``y3Rcf&O#ew){Y zB$9o=H=LXm9^v!K+{I~a>qF6JzKHdSJH)8U6#q)DLb{Nv%n-P-=+w0>`?M3-t z_tP&P{K0>XxDE}SH0X+}<{I{i3k+N2Nu@`qk3Pt@Hf~+=gF$Q6FCIy+jnHUXE8gU0LbL>a%v|t@ zyan7fLgj0~la>#{)TCDZ(gjPbsWdR7I4lZ&6}mDw#?MsD2}4&)S`g#g;nFc#%IUgT z2fT$?`sCF!si>+I@i*waE()_r8sYpcreW)%*6tg?=JZ8p8W%;}IOAo>nLDRH|FSbg zGAox@E~g03vdP~5&JdL=d&)4rxF~Y~er%HL_8*?pBLf`LK=sAQWVKYhL?AfpadC&1 zgE&*W&1l-J*h)}p&5JrT)KmC`BNAd}`S+$;2^&5#7 z*CoJzT|{Ok%>YbrRk$;S47<>G>}#TilGtQ2YWb;9#k zS1ej7h+wFwiO@W}#I(ROTEI5;*ZHOvOoY49D*uj3#TNpX>nfZTT(-)({`hfuK~(FD zar^9ivxn{5?yg4VTI4(>85g;hMdDiK20pB+_8?8cX`^83DLqqMXwj*+3J_^~0?6AR3ugjo!Uz`f3nSQ@CcBvCcHOb~?at zYc$(IJiNsiWkoVNr&%v(v;f^}rvp&Q+Hv=gXIM)|ob8VeX|~@zVYa_-Cd)k)Bci;v z97yuoQbUk?iW_p=6gC9beOpgn`)V}AwXZZFt$oFYum&YPWHl%Sq8j|RBsKohCaAro zEpi&v35jXvb3jTvWuB0BODq`;3VMiWUupjkV*+XmJo$9Lfp`W#x}>wGu-josHanjh zqPfkH-*`af#uIuNzo2n03CtqGmJB zFVLIQ=Q2faq*Wh;f*uI9R7iy?ER+^%^%vVvw=;@73fYxQEAwmVRdXXLh)AX?DDRwE z-+q*7Pg2G-Czrm0gf05e?Udn{@4kyP){E9Uu2LzaRczkq3z+w@~x<{lE zB*Tj2=nAjGB+yn07H)SCaCK%MtrThW42_A5DISWFsg@;&!3M?CFxxY1Ug{}ViH4iZ zX+iXNWU1)X{0J+aLy7zDMNq?D z-MkpnGwF3xjwifvx@#x9vncos7gO`viz=K(URE7n1#cZd$SA;U4%kz&pi+sIcU&6| z(O-?E5x*w{CGX(ma*!D6*Y$O=0!?Uo^on1D*act7C>DXwLyeGYD3P(O^4^SLy*XfQ z-{%g5(>kN4mqe`m3VG+S{7O2UPF<0Gp@{I}kb>^eJ z^dNoo!1wl=dn05BS?Z~TW01Q@60&LKb#s@^RwJ*^kl!q}Ol)5w(y=?avL9o7;5IsRAF2989QwaQ^D9f!O_bG!d7)r*qk@$c>a>B6G5c01vc(5jTY?mo5Lr8nN42@ zvEz*ikgFvTGMr_k0fR5P$|(*)|)(_(h z_~R9L>;=4YQ{glu2C<7`&NDg6ILk<-pga)ZLPoq>;LG ziwsL5(S4FGP{A`?IFOXcU3Y;)Uvd|vY5Krfr{P<3bD8OenEC@fL17S5q(83q=BhGm96fO0r3x>{d_9ZSz+Uy0W}vblG-fsd8ey{Ds{4aIi#p2Bq6o?!6&3Q^&c zD#qe%j|4=e>ji_aZsXf|;>*z4q2IssM?SMgGWV|?c(MjwE3_qnURd)=zFR$*?jGr} zIbZy?BU|SQ8a>bMi#2{eJP&o2%~(529j&E#1rcETgX4VFKH%ug5uKvz#42J6#AjzH1~bx1qmnT^K#!pN-G&Si7 zuwmUgW46aA{SXwn`EFf|65H#j@yrVO$Z3~tM995OV`ZiGTEe$l@lpeb+0xhig0N__ z94-;yuyQIzZ5--nbq;wNdUtTkRS-2xg12eHRmo?gp&{`r$`%RM>(umaI(J@VpdSfQ zo=QZ^vQ}__HOpjID2GD`7~>pS!7JRw{|yX&jiiE5Ntsnm7Q_XKl=$r%hMTY6v=!lDya*=O2EFWM6*isu2+U|0 z{WMAwfCqp2;a!oL;=Bl83SR5K6Mf|+B-anokahDfF3GuO@_4PR7Xm_* zE$NJ^a}mcJ7;)%13#hegqa_ijIO+xzVYjy~adeFOA0I%aczY_^MwPc!V~2Iw~oPot`jv;t+= z`D`^se+xSLdsKRbshXnwv5dYpKAQ)nN9T%Q6_iZ&mcA$>^pc1vy2t9mBPNU`!0=^$#Jr8uuVCe#8-&qdWQ{Rb^G&yA4f3_pwA; zh++%KhQEs{p#>t%Noj%1s5qf9T4K3GC9aHQWm}nx$$K!@MzvO~cD013s=}628MwrM zwj?l|Qb65Lur!j;c1x<3g4c^B`i>OkJ?S>kC`N}vMA(8dk~&}U4KOV~B3o>ZLx^1i zPc6ynUQw95Pxdt}!7Cz)6(Taj70Hq0HmlQ;tbpHQTp49lCE%u=tqd?d=AKQ`;!-M? zW_xmSNeW(3wm7!}(-7W$Ow^Jjsw$Fs&A_wu#@*St+Zvb8&5qm>5aUXVq=wfTy+`A7 z>%0j-JTk}L^SygxAF)sznJ3U18_sZuzF9fxWY;%%2KDvMfG2P6-@oWP=O!Xy-fX{9 zIK%mezLBiwbNbewORyy8DLNzWIHw+u5T5Z%dz3Wy4(!%0!g-FkS`uMl5eYbpb7LTh zN_KT0QnRO;2?tMH1Fyd?qYBN68jNpTlBSDWQizYU)x)9Q+-PrXe2!65pF7bH7TimV z`p?flC-zYiG>`?(E>hbjRphN2pyZcdg1QaD48m&XWnCy*6=Z-?6g5Z?VL1Tl?Qso| zu+uO8_U-AbHzz1TM?_&-$iFx5s=Nx+U%0@clqNQYS`P<{YIPAgNP^Mp?aIQTyH-HaRFsZD| z5{uR>%r}cnH`Ly;pOQ$47xAjUFyDIm>8ZVCKf$ZlQC~;FYuMfX*0kEPPwP=}UW+2? z^R)2+nSO!(-0n1Pc0H996}UKOT#@q%=hJ_?X&j*BvMA7Bc1lWp@7*{2%vnJ*f(HIz z48jP5fyJT0gP!SnirS{1f$!R8BKnC5=XRgWB83{s?N`wrg#j8^WF4SEfJOr}2=@1# z7d5ne{+3oYp!qfis8{0!--hTJ7om9q(yUA#L6DUVW3gAGR>rO?A+*No| zU@2cV5pidMcS5X`oE3aA+8ghUN8^X1@!p7*mEe~UIVTgvLBepo;)*7eB@KW?k2O+P z>gt*9JmK>MGuPA(qdmH*33T1*)z-qtB61Q3X440;S!iJNM>(B?RZ;l|>L4VH+t6>L z?6t$Nzf2J9qw%B3wxFAQy@Z7}k6rtOFhsD<>%u)IYz0II9`aonnBIs-nN*mWNJ0%U zls?dXvgi*lOu}l(_(41|v$WeZ^o#l(mr8n*p>9T(w~k{{ zN51>p9XSmN-afpa6GcQx8Ifqoul&M5S3%6N(MaVFTz4P$?9GT$L?n6;3G}Q_p{CvD zlUh~8ojx~{DLU42m+l6R+nL2hzd)~_pSAQN9m~&gc&>-mZ^q+KiCK*cpY1w#@%~Mm zX|PqQXax@gEQ7-lwpw58L=|pE4*mSk7VcRI(;;84g)o|y>ASHw>0i@z)M5#JT`*T33L!8 zuJjorI9tY(w28O^qBtDJ>*fd~5vR7U+gVEjypM$GwW)Y_)xFDAx8|mMMl~Na=c07C zgUc-k+YdWi1sh5|sdf;4{<*tFsRm-*wSV~~`T1w`2Mge3UCuy~O4qhK0&4D9p2izA zaQ_CP*)ysJGeM{ELlSrUg9C-Dim=>hRy8cba(~m9ZbD~RJD>)5nDt@JP&_!(I7B_mVD2@sU?nhuGX%4Z;jx&C|@0p*# z`OxGDGnr+`TP3iMx`Kw$P6l~(LU1QGQ5bn!aY?j10(-=GPQ28?ik0b*&fQ|9%09P! z?|ne$ zAe`sFRlWK#+3YY^*vPXbzHr7UeGg}xF_Kw7H^EDNhOkgHs|ze_{yJlM!%{XEz0{Y% z3D3<5QMB}Uc$k6nH@f!(+8&?W)3dqvk~0z=3u$r+`^?5j*H}Afp1TX!fo%go*8j%S zhNX3(^Eb@DjnTjUEBf=-kG_039^cnkJV7~LJWN&Oh}1KIb@}S;Ap`k~Gl=wWz#h&x7Xy75a_S zr4NYwEmv}nf-ho!-8BjEj_2Tgv)cvagcQe;Iyof-?zw=NN)&U~O%NmWX|nGP`mS)B z*sS-8e%=*l&p;XToR-unX{A2Ei5*G-1)L#fcuqdIgUo$0x=6~ya4EEgESo;2s3D3b zu}a)EZh8cnpwnyfAlps7ss?=6>&eE+*<>WW(2dYzXOo(fe2EJR5=^k0J|J-~-TNvX zgPRH?SFI32=xM9b6@)F+G8mmnO7@9{sO7KcFoJg*tht&$)o8(%v}U*;F+ZiumPB)@ z-@Sf)Vt<6{3J)lfM&XwGK^4-FydTU>ZnUGCfP*&~qEbmoUN=P0M`f?ume-u;Yq| zqo4G&N3U%c4x-F_dRMD-0^ljAa&BBHecR6MEz>jOj0?i$yG&9Q(u~-t3*1~BU7Zmu zh=A~gXS5(wKkpY|`q3vFGJN8>A`RZ`qFJrhX8qhIPmqkQ^J{GR3#_1T$gL02=3EyHAwSLKJ_ z+OSjUI^98?!8CcA6=pdWIpVV)Nv7^eeXwwj6;9C$bK>W`cf>)QPZQH=hi4KRaHa&E z)rt%2uso7mY7p)<-Q-t%(>9_7`T{x52zp zh}SS>AD^g~{qW>i%Zcu#ny8QM0vQ82_XVz6%Nu_G&@0@tw4oL)=z5DqHdtEXsHpNqlELhA~SoJA#_D% z5y1|ZGT^jk?vV&{A(TWchzne|LA=o>XlfZBrViY*@Er)-`%^0hSk#9avdP36wGR?) z+txs$O~k)3nwh~3Oo5BcW-E=Q11oVPt_~ONZ^@q3A#WPIXJh(b-7}FJ_N~2OOhIAz zq{}gj3c@$^n%rLripP0FabnFW@3+R$;tF1n0pxv9yt9(GU9&KAf|-Q|KW$Ia4Zq3d z+s4bb4%;0
jb#L8&xXt8UaM#`2XyFmPkH5BOvjD6X82WBhOZ5}0Di>S7nZSd7Y zi&55p-h{Q?3EUceqmh0XEN=O%ZK{#%?c8R)Z(L}P!p2`H*)`mNG!{^mSq5CDh%Ba?SK)<;T0XO#tX+*-67-7s?e(Kty9RB*jl>PQ zsQ^}M*62B3pl1*|Y1_P!1QmQSFPm(65A>n_QF5jW7EXmx@P%nOwe7EB3iKbuF?Hl^Wr(ZPANxG0U9L#rr~+3)V7rMHM2&+GSg>K= z8%Un_sTnvYxkw|3#rNK6;Sf@*S7{zP^*k>v3CbOZ7<)P_&AS|$l|&%=&bjdHA=m~d zd*kIjlc5d_&y7E{APX?66HgmmkAE8{kN^J1`=z{=*X8x{0lHt(qM*`PjgkAiq^wrp zZdMEVp!K?sGJ^UP=>ar5V3OjxO#n6$^ecMZmi!7yTOsntvsr?AnPK_S^^1+CiX7SD zrDY_c%2ul1A-d#6U6LmSrsZoZ|2F4IHfj|VG!Ohe7uGfz5Zx>zQoi81dA{?M;QX(G zD)NeDWbpTp#qcSKqf1ASpK22MJ0lbg{b2kp4aXF{WfqEG-X#0x^!bKuY0N0jGYHiz zH^<4uJ20ZBgiV$?lK_$pKU`SbFmKRp?X+~_KId&Vpe;8e=Pop#IP>BS& z5)`C%0B90em>Jl8+x2=AoN2NXL8B5$W!})M$*!BjtX4jL#hQhQ^mt%4BZ^Q7Cmh=@ zGzA7*hrU;A(oZw`_|P};6w|<=b-mT;BN>{uHrrTdqrhAEw_&(3Dk00_fj$m|9P2h^ zjnc5e#Y&IdAa3rN4wqA6!9hW1vQQ8)=b|L}E+TI;QYrhCM?n;Epku`-*1Oi$#aWKP z@UAa4xiVe^JU87+o60;#>nuZ?eFee!!w@IEV&Q;vQ6#^##?(lB{t)Ox1vX*(ZU6*Q z#~Pq@r`4BF3sdkbJHN zs3e$4pV=hn%i6TRFv)DbFg`xYS3v5Ddrz%OBNAt!+n??os2$^IQDcFbBEVs|P_$@h zc1TMs@(n|x5`E#46u#4d`uZH0JJBYFx7l~Y?Qn@-7;hDr;5P66<2m29rQ(n&+ITR= z7J?XJ6OyOth&iN+SG?eh)nCXebPI?Y2J@m4@TbZ7h7{ioiS{mPejz8J-udXzW4{@m zHPm(d9(j(-A=^<6!O{lPC|evGgNpfVo?4FN)R^qOCZcR9I5w$2yaFViEZHLo{znqo zRn4EZnve~;5CAq=^L;P6&`hLr1nB_?WG}^83sq4_l z3u`pEAgh6&M)+8@mg$-jScoYaz~9a1-lwl`p1eVSBr8EM=%lq6Pnmj$#kk_)qTo1} zBTcg)4Yq@y82Cv5zqK%?-WgPT($tEUsl;#3F36S3zc>sU!ufSS){+2(fG>N)Bvw?; zsma@T#9wmd*34G~-B(v_qtX?X$IkcZ}rpQ}`mb}QN-h#wvAY6`Zgq5*V z+sw4~fsrP~t+dAdm`dh>wT5wKeOxu7}P8;{2+`r>kup76Xqwn&0` z&%~)d3wmaNM|&>gj=d$jW``1T`%Gb=;pdl}opP?C^u|!|MiTSPg{waA0b!G*+ic~5 z&pL3}rzUN=Rc$nCB5>P=ck4kB(_?tE0+4s`Mg`~93}+X+{=%iflixF{g|h|Lw}azd z(ea7j(wAHjAF~ho_Q(y__4?*`7r^O)1(%XmL8jgIW5`7_i#9{;UCO0-oL79HL$tBFhjUjOTv^LiE}U1Oh?BVqHQy^odoAT_~Pjj>aQU@H|LJHn+50LTr{i=oZDOcwRb3;o zQB7Zuq1JdBh4tuK)=#Ie;$BcOO;i610+GUD`Nil&2qCXj?pqdc*i<~@#T31Mab7*^KXv+h?!l+4Bcgy8G^b|G^ z5jBE80E>qbP3S~W4%Z8Wgd@XGHvUOk9F16HTRqSh_WIbLkpATX#Q~p(>lB`PxTY%N zZYP@+5n;5Xi)FD2^m+au^25?S&YOy)0HAx+CPIf`M1@+g&XA9MO*~2-aZOr zM${D{%w0BEDN6Gn)uY^esQK9P%#05FyrB8%#`|k@6}mtTO&v{(t`W|0|(t zdZNEN0QVojlOHL1cu>k+v_F=+Xj1N?$#QR8P7dUQaO&}#A|$poGzz^o5o_U;Z4jG@ zy0L~8jj(|a{O32@U@pB=t>v<+i(;!Su59Jsz$Uln70vw_7a1{6M%V-WfbYFKFXrnp}G{#BJnf{w9W8%R1 z%GwA^%1-r9AB8&ZbH|R=_GaqV*aKcKja|?rP*&6>tWD8<7aeSqJFBXA(7K*_bm||b z^8mq|0m$d`!ucg;nI4!=GmR)LUpG6IgY!x)iMXPY?3#CH!el$0M(z1+zY=F)APe$n zKYgnNwqA}|l$`>Mh6G(=L3u5W-I+3dZXgIY%!GvQ!}C!pvgLRPODvaqC9$L#2qmG% zD!eq*jKTY%(;z?*JTtwau@3Brxzo1Y^r(k-?6z`7t(H@Cz2jKuIp@cXW8c|MS0&i|-z%*-E}1`xcsw^+;3^qd=?a$x zyenxwPdo3-pvJw(<+2N~!9K!sPipoPl9{$0cl7P^>h;s98$4x@Mk*i^n$s}|L>aZY zwp+0d(-G(`QaeiEpg>Jq6!B{(UU>)Xl&ek|HFyGV_&F7VMvQaARiAUQI5%E+-&1mB zP3*AgwzAdEpBH>V*|{EM6lz|pWr`VRtCH8F-=f3T^4O~%WHEIUq-X0aL=5c3k#5mwmQ2i;qd{uA%5wWYI9}vs4{?@^5b0lBWIB&TCcSMarw@&8kk* zK5$N(5}FYk$9gTA7C0l_cEIvbgNwfY)IOe@MDx(#gZNW}5z#SU}+)$>dR+(eGa%vvoniqI+(-?bCqy^9%Nt~CIonR@i;34Wt@X~vu zbdn8+DJa~X;j+oKuryR?^d@c`x|Uv91rut4ZKA$C=50(P+lp7&m@l#C?Yvg+k@~9t z$=sHK)OMJHzRva5UsV+{!5Q$NdPN)A*TyIsQ61+&(!!o$^(1)w&YHiQ#e1<$2Hw`v zb#kCOPAW*fi=tG$UYUp05?0e)Kw&yP%Rtw} zU(ZhwA;Es5(0#+kp9O*7rV;$#S2PLhhCQ5P<8EydyAFHZLkI0o11G%%UZOjV4?yHC z>Y&npkSW?TUZ48IoHAU{e^|cR$45liTwP{KCAc!)S3BFjtf(1qP~FsOUx_*25b%`5GOw z+)LHq#Tl!@YOJmCV}ff$6!@&N{KjgQ76tHuaeqVKz_UX#mU|l3cmo>v>(14F+EaLHT`$Lp9u{pbA?JmAg>pGD24XmTL8 z!pyGtbR%juV9)d&K-7}wKBN>hM}+0*rgrs(9=+3NscSE}@N&bV^95x^VmOJR~m1D+yy$=%2FrE|g zc%Mx6^6aZ@kKoDP8 z9(`$tXXelh-=>9=J;Lo8(%ss%btHL+o^-P}gw zkp?(mtsI&xNy-_?Av1WR4AoOO3xaPEB5y%gD_#qnk>mV`=&m}%6wZyG)*DS-{s$&a z76&-R2EucfuWD+(oYUMZHNtnIJA^f$ZDf{9>LDtSXOiMNL8DJ(G zEilE1;hQ4oO`^Yne(@0W*a27RzXoC;ma<)HJhgJ1amS&0VbbA_{`#ZjOpmu=6-`3w z3tD|o1f8#%W&lKx+Sk^P6>A)=a5oVT<+ZX%Mj1>H;H_de8m;izwco%u8$nlTv65K9{ z^rNwtl-@su^rLYD?h~YtJ{UKG+twL0jgQw3x>cT>g3bgMt0a`SAv?|_-Fujh6I@i3 zIVF$b3K3@L%%;n444{;?*7Lob<~&+eJP*jG_w>1jp}Ldoq|nCyD|BvydcpJ(lq~!| zO&&7D=kq@`#hL%-2pXFIr<$5NUf8(Y4WefW;KOStTt(Dc`*?-JR+m_e zGMp_*Z+km9UJ@^J_WH=Syj~~qXY2W@zZ+Ej-g=OQei-IJ2EN6zgIcy({1$a*h@`C4 z>WDTO9omQ(PPDm{H?5q(-7rT7pqV0N5R_!S<~!JkfIY~`6g}L36nrx(qoDdutckQ* zdfdvI8!D1E=GEPhR`<#2kv{ohR$Uy*e>`N)hqsIMGi~7iq-`kCz(9@GlY`S{EtDyG zK^IFZ#BU z(^w0@!M!T@3hvkk4zSIu>eScqlJ_e$KSDBMbHjwbM<|FH^<*ZFu{2dQ*w>BRE$^~5 zdiCh9k*%>6J{bp3x(LX}|m{dfnX9Abxii?Z>r>V4fT40bs zT}&S?3P9A0z)6gk6=(1SAmEgcJHz2lQz1SXvg??!0|H9~#HE_mP!O)DB?k`hVMWj2 zL!XyNgX;#-pz?$6UpERBLT~hG7q~Lc+!QU$>_>X*MX|pF& zmMVmSbv;3KhFi2vrkVy>66EYJV0>+Ib5aGzU?!MTC=r~Bh{a&SFP_pZn3zWz`Dc&Pu= z|M>EHoPKpZP9I&5)5q77bnoDLlIs7yI=J4u-oM^Y53V0xKTHp&e0O z-of?5@%4dyFc!7*S_iCrf?SI0=%Wwbyi2^Zy1sh$em`#of%jE) zX;aR%upc+rw9oeX|5F3$Pq3zbS`q7bes1G#EK+s2$G}i+A*b>(n?8KF{~phe*q-K` zPtgpEn^>`e7?q;^ce3PCUlrM5en}{(&Fnb*4C+V_`@ie_g$vvsHx&Xzt6x8*~f zf4b#1Qs;*$qibm|eRyE(UoNeA>gQOGIZNwZciB!0(`g;p}0=I|cxybHd zU%N`}0pGF5ckJ;Ud;Gp{=2$yIFRj;<)-VDlWtdIk0x#qSFWZ2!OcHN#c@NfsN?8** zdc5~#LY&IY`dTazf*zAE&hue1S@dBF^Lv~oqclh#%s^LShZFhaPr9} zx$B+>ex6nJ`TcAz&(%_Jr3#XJ(MxfKi|c~p{2X5rffwX_GM3lY(iYdMB@ywb2luv_ z+X&&}3a=#k2N4|2X+aG5k2m4DJikx*c~%RdB{0g*HCNJ_j{AB==whilYobq=Jcrc! zx&0(Ty>MS7n-(fwp75}jT&zCcA{qGS_e)$=BtO^taxJfLwcVyx zf@iwmqYdxA9?*GCWF}~(xRBS77w8;h=WDgJ`-9)M#~Wlesz~N(Cfus}@+%;~&n5*; z09^9XW}%0We$gJ`Lh!nZ*=EL@%r!j&p!u#Y-?IPokAJ$?FKIVZ*)=xy?F=Nrw)_nz z|2Y5qA0PawU!?6Zi1;CH7|pGR!HBxz;^JIo)p;#7-7<&IeXsNQdx$)m9=1u|gWFB# z_j!jI7N>iIZM47Lq0@&$r{A7_Z#Z;x3?gt%+Jq*C)?{vXX|HTvOGrZp0pAhl)Wb6+ z9jmG%b$w=FKZ=l0M;f`X(s~^Fj4jH)Q7kcpX=9j# zB)cXlwvE|n!W}?=YhR4*hH7*}r%b=BE+G1`X5JiWj2Ym}Y7_6}C|@yL(hQ={XIPs2 z6sGSm4V*P%pgc^=<9g$BuY`u-Qx-OzqFJX#-m z3A%roGu3%4bY1rS?Wj|)6>v@c(DvG}Z`RSp z>l*2wT+n}*C@*+QwdvtwT^+Gkhq2}YxwlJrc0ntXM1=@N=X$Kfps7nxNm$I2_W5py z5JNn%u885wSPnK@w7a!C-#Cp2Q2)6^u{$Tmf7V!(k1GyjT#zFsQ%F%vsLkOz=q5}o*e9VcufDXkH@@tBolZ<1!!{4s()u^=CY1+i>KFAHM%4yT!HvmmViQJdh8$r8(-oPOC*g~@n)@X&quCLF)F_)M&*u`xML;mScy9yif~&jzp>f-@($zQV`;bc zPx;&t|D*BHL}j^SlAqdK$r~;9y1{2;p4Vgc`2b^My)ODl_IbnP@9o=nQ$r5lf4l&G zK!Lx({tXM?&hu%kAvRy8%d`(G?w)s>tx&;n?w-hm9Q2ea9%cf=%*?}W20R<;9*2$- z+(Z6I6Bg*7!*|cb8Q>-gOphVS3QCwtG9r{)C*4U$Kp75O*{h`cxG7hUX%(1k3ZCI& z$)$3I8Gc3!Vw1Iod4OT0a?~EA{eXRYBGfxUfDaDJ1zEj2kmQ1_HYI?&Q%c{-dUvwk zZ&%hseR=yfQN5VtSqP5Mxu2eH*$H7Bq2Xk2vs-jjz5dg=0Un&ke4kjg*0CwS>kP>vPe z1*S;Q+&Czb(5NEy`pNZs!B@@{rZsRZNMWy1L(@X{$W%`9#!l+!%zn}ApMW0wr*c! zx7iHJ7H5$5+dqcL>bLzYhyIddaYfl$mSi|Y&$!5l8SfF6%NfTaM^DU)tGWN+UuZ|< zk}yQ)C>$k(Z#zVqdcL6Hu?h?P0A&a-J(k4c);&~);0=o-0 zz`Y2@Jwqk+UOD>QJQwcXkoC#dq{1vWt5ueqAxK-Nud^okTJzAnTY-?RZp7wj6zdpJ zcoX>0KOmi;B*}PTMC`Ol5TC*;$B9utCgg2a)H(V3e~s?vzvGL^Y>+?;4f8Hb9p_lCCOBAy{s2R6|=d#rjkGY^2_mc`BP15a?Rvi zUYo4sD>Ehizq!1wq~d~Hiz+*>Xhr7o+WG9lKjHM~fBjTrxtv4B{<*wX0%v3{frhMk z|2@)qyqhf0qa?YL67HmgJ1ODUmJ+_>SE%4@foi5`p}RLTurDu~x4F>v0>o!7Km2nt z_BT9Du?3e2oGIL`ny;7VY%b65YxYQ8TV9PbxJ#JS_r`Ww7m-p@ZHMartsEcn1#H{6~`g|mxv zUKfR)!~mt}mtXuw-3uPR(k!>F6>-0&qnG|}<3$eBqZyXOJYFV?&tpU1b z)2P{j#4Rkk9H;w_?G2z;Pzwtc7`9$NKSK;Fc#Jo7P?eJbdaT+At_f7uQdth)d7pw@ zVRSA2RPh3{MasouR9!4anOIedk5nN?!|Gy@WSnX7z$RW+g?vHT_XgS&*?+u#el~F_ zxZX%2B)VF1aJn_CVG;&YX;L`r8}Qq|JVlkDml{)SFq<{nF=Em{<2*-mD#(?VHH{pi zxP9%+0E4xGYc?;bG`Hv#nJqcLaBsmYA{Z`=8^I&{tSZP%wt=hfms*2sHs=(&^{@Er zF_HEK?)Pt;?vI@lh20Ybk$6B^5oH+{LQ4ZH-Ajp;vLdFP9c)U8FKl*YD$o^P^+ofc ziYvkt1&25b-lb%29zQbrG=+sS1l2FnU+ofk3G#bjOW=ISDSC!!v5Sruj0@uae#MS8 z5)_4IV_dVIB9v&p;?#(I>9Mp$`dHS2Ni@(XI@r}*2O6061H{#mh$|`y8tA@_)8ZV{ zq88+Y7c^Uiu8iT4wyepSdAG})ZR7d9B&fu1X<3)ex!)g~3qEcICwa{=8lppV^7I8F zEaQ6PpB$oD&GLevt0m2rp63JhMRDgTN1%U;S@y|4YrnOQbD$`O8BeMY8umNTW;qp47V!~VWD(@YP^;OD6ZXAxUxVX@= zX~O4;T9RbO->&5p^n&3_PIEBmvqg(jNc|vU763X8kAzW?X`x+9L#^Atx!UcOY2BjN z$%$*vL~sUpp7<*|_-=J|i5Y5Aa@p3c+D>e3!_R=*E7KUVJ^_L znNwGr9V`iNFG={GZFj)INPEaGa$1e}lilUxlW<_>41t@M=K?uH1ZPVSjKXTbiom5z z=RO;RT2*ko@4e(X(SYs+O`7$vg;TUY89y5DbP8VU!R}A|6aA>;bfPgF2oA5vZP#xr zq{=wUNTuY+wv~x}a)z>n`!*Vub)jgY2jj3y_D#5Ph|#k%f^ImN=N#uVTwseZ8N`IbnBEfS`Q+-}5g2-D# z29OeXKBt*30ZRpBm4FO^2I-P3&0lz*=e~hESg#Ez$D%$l0HX1^@#s|vzdID%b*+w1 zc2T%uef?@6)22bZOH?X-?H!-!4HZO6Ev+car?&KamXEYsrgzIa>B6G5a#`QOs62Ir4n5=NvnZ%nrLu&|{ky6`5Z*(x1g zF9nfHUgT4>A25F6RNwK>k|`k4t= z{Bfps@KI-G6l%t@Cegn-167YCYgKOngE+L!ns@aP%Bp&b_QvBf_-5WWhtDsl|L%I~ zw4U_0oQ8S%Z%Rsb`Dn+Y5@YG$HAp$ARO?`eME$??&tk(b&^ zPYJ3d-P$sb?bTm$sa_h^va5!$G{SHgnxT<~&(N0$7EKa~9IdykZRRR~K{JsTl(UXZ zq>Hvf)7<`oki)`5*N`jc2r1COLoq&|{5>uAi{D7^>aJ^Il@KiFp}^Iw&J+#y2E9#4&g9r@9E^L^6V4QQ8|X;p@>{{{ zYKp!*IM{DyENXVRq2wiJr<|+SH3~&wL$|u)LVteVFPha(H-OA#k;8!PtUebar$Wxi&0voGy zta;L`$}3i^^nyO~4tK4k&2WWhw4jQHIEcWRApEU&|M2CB1VJD zD}{Isva#2dQMN#bC&wrgBsX?6*-^1>R;)G7a9}lA@5S09L~IZhC|hXKlduJ4ZxgVe z+p8KfM=l6!ar%4Xy}ila-Z*?~?sAeIq?6>}QL^~z(N_e2^>Fs(?PUQs+8*``Bq=b% zU@sn`Ov{JXg^Nd4Bv>K6kuo&c{OjV?OQAD{57rQ{*W)&ssJ#g3Z13&C#o&68%$T^hx@ z#_wihpuQ6vZ0^}TWiY@jPL`^_YuGhk+M=N#AiQ}%z7Bu zkPhb}6LC{Ocf4snbPBMs3%<9=#i}u!xaj}H#`FTRc2`Ub+V8Sz_w|g!CE7u)MR1qP zaEPAKLJ<*6M|W9htZs#+conFX3;k8^y1;=~Ao%gHjI_FpD;W`_Q>E#1EflrHjDxe) zx}F+$J}ZVCffQnm_TKjz+_NnIVJoH;A;JnCZk5APTdeylRZGn$0v36P9M>*_1-&it zrAt;8LQ8D;1Z9gP`aVvhpJw$*K7BlX{LpJs4ZsJ@h(7gjQja%1A|H&^XL^q}Vu}WC zP+(C5f?=(amcKNI#3#XkyV35+%dldDiILk*pF51k_X1yX;WpOxhzwsBnd^rLJptc5 zpR41GP+_M}M`J}zFCGCmb2%ABV{4jbw4ikfL=#v5_z6w+Mo-{Jx8WSd8fp*Rtz?=v zSwslP0aHIq}Ftglq6oT@E^#HzG%a zUKfMgIv2Q{MEx!WH@g#jGFJje%x-ic=sFK<={DffarHS1d}v3359TD$T9&{$U<=29 z*x}y5D%x~#3!{41algmtf8%J3NgkSa4E)=TBZkmp%)g}}e}fT!jj4BQGw&J`?=8%` zy#{+5w|f?nWuxcQlzzA63!(WRD}rkQL$f2`=!rezhLL3sxI?@T7-6&|MSru?r-w%` zo~C8q%g{tOo|Q8~LBfvn9RqCabX)N?vE!e5;`wN+9-j1J8$46+Le^9S)&?dj2_wot) zp#DHc%4!=<`+R`j)&FOs@6TqxpIiC-Z1DH#_w`xl=hN@y)AjJ_@$Tt*_O!ivw)W_` z!JDV$$G#)JWBKj%)9E+)t~2+(qi@bfF^|JJ+3b(AjcN5p zvudA7H8iJsQ>s_vUlO5c21-ZBJ{Ubt$F3K#@fum(C?0I0#5{4>U2c;EXEIaXB!U7CPUXDimV;+`J3 zlOfzrA@UYOxIbqk1eJR8qW>ft8=^TKhB7BXm82p9@&+*f@PyaqH5m#(CV+=@)@)3M znPUJ_bQE}rWL0hSC&2U=bZ}lp`Dx<_(G>9Yb4&iF4548x;IJd(#%o{|3liWNagf&yAWj)A9L2^K~4G?^PA*38` zcv#G=iE z$rR~=ZmJfqb9CWCK$im#zgq_a^^?Y)K%UTXZ&LtIG+;9{_n1>miZu>3oPVE+Zy40J zgyI`4$uUefC<|1c>qg*0r#{9|PCH>)mc!%;I6!)~71a)aKHFYEi@dbjspq;JsP5=xxsp;%(TAP2?=d%@ur zarl(K>_UdfFd8O+DrPqPWoG+`F*aTtd-VGvmDWZZt%+4S2t$=_7v^-(=qO%;KMj&J zCrO4K(g7XzT&K4&8RwhhEs2<^ZU2f$)&9eDveVi#uwp8NjpiOX5lo}~+&C&-Y0-=1 zoeW3*=iwqS$aQ^*8bSWHJw_*WgU{2u_Yk->^*cGO1^o=7JvV?h>2zdcjnO}$RRB{< zHvj$WjsS8&L^faS_La6AHWI}z2(vYpq$pFl99gHh^m5|H{DJ)M%?@nq=n?okT=JqW z8&?BJF8sau7I+zS8X%Jav8UM!C@^iCoY*+J*X$O?N%Gbu$9v5eiz3pfKw2-XaT}H1 zMdT${ppSzTJ#&IpEr>)pk(r<~eHoB~U!nUXU8F`;vx|PJiCFC##D&C`f|MGHrhdB? z=N>d!mcTnoGp@epKDF8h%(IviNf#{S5fbRz7(>mL>)(d$3Uz~_r9Adw*=(M79zY#$ zFwdHYfO>q?{<%EOtb58?=UqMLgXVtQ;1uVS)+$dL>Bd*=lvF(L_S6527J@glI-KQZ zcQ&KD{H*=hdYsF)Bzb=B29M1R@0^sV%=je{E5sRG`xzFYmd4~>#0-`Af;jfzKQ!rZ zDb}F&4@m-9$G^^*l;ZM_=L=C~>AA|P^bbF8_BQy%aRz=U=^uXf7O`JKeWURZ{XuV) zuMFJ>Y>VbvI134;d2K*fyMF=~kfwA5gkS%fI=XR`z+6NF(?l2jSWAU6UabuCfu$n< z{ei!;&w8J(Y`Dl9JO&*B`OG}b`h3i8bRS!so;)!ryn>XuO>TLOFaFaMO~#MM6U!tv z&v_1Xt;VqX-RsvUAxLyzBe+daZgX1m}g8Biw1bKX}O*8JMw!zM$~AvE;A!2;B|58jA? zZY}kUUPj;dA_5V-?qE&rXZA!jS^tOVB_X+t5!|YaT<%gV(G@9*_A%J>f(TmY5p(-D z*<4GREs5TBKoj#Ll$w3QaC_$(4_mO?{h$gfsQmy{t$jl;eJwQ)N~NM4m9L+lIp+h@ zRWNQ3Vrt&Lj=YP*N3y+pnPv{$iA>S6ZTcM!(W|*Vay=|q#s%T>U9wxNLI%e)n2THA z8Nq@GxM`k&NyLrky}d2^-7FmVcZb_{ZRp&Lo_J!&O#%?x7h;o+7%poWxC(|ThT%T* zw^6O&@u@JhqfQra&yRm``nMeqsWYR;+waP&Iz-DZit&fD2_liC0^h*A}7bSTG|(= zom^ct#|lLrdnfIJ7)rxAw&3iogR|H0+??r~QyAMcE{bO5!A_Uia+0HgAUIzQ{8Me- z8B8rnDM@ijypP$oM^2DA5rkzVC`@ogy{^+M?){g`$!KUd_^cxK%B*JvmCKi0o$4VT zI%?6@5*^#DAB+}NEQ*fs#?z3Ykm(uJQw z!s!`7|29Vd`mgBEUqAZt;dl&8!vy7c^+0c{Q*=0BUw@4TuBX9p_JmOaVZ(X=2L|Bvg)*RqAmu&?ma)A4( zF5U>xJ6<@=YTWATne<;h)*N?yf&^R)h&Ah^YLPiWZdU&w^_)!pt*LkmC-3f2MPJoQRQS_RDHTR)oqbj#2 zOM12LdYQ8^*oIp6euuedT%f-+hKGOq8cov4BOvwGH_Psj%|>C8Pc^Be&?UuCWRlO? z3O+(( zyuls+a}(k~9ecMt)vSaaZWicY6-Qu5LYG)jUQ2i=2<*StN8Dc;M{zW(+dlMTnc*+QpBZ+{Gp!EQN4*Z)6z z@4DW`jU|fqZ#@ML?b((x;!PJj)}Hit)V7mo+D_D#-07K4Mk%sDvZ7+4QUyv@+uGlG zjq`fvNzR9bI|_9pDOq+{KWwoISXj6%EUfF6R8>w0IeHwV+3@j?-+s6MkDF2YczpSK z@a^|6k1xqT{_7{2D?f770)G0%e(*eu3e74O=1z#60B z!lh3QkJwJiP@A_{b4%Zl(&b3$w-Fxm$7x3?BOeS`!&Wf11UO0P-?t@dlzZuVCRDo9iAw;)7y zI0+RY=FOOi83jU?$FsbSX0e7VvcNU2}-8i4~M&vl`+gb*rELXg$~T(B=%*y?DvwX` zjtiWmm#1l(D*Vk}m~NVKPm6Di=RCDG`;~19y`IQDwv@gsV9&iq-C5qoXcMnz6R+nF z60c`tPPeK@!>6~c+e9+@tRfk?htn&c>QA3f>a*2huHP*&#t7)_2=(P_4~_q<|aOn8GwC{a+*Qvea^dlet%3CcFfGv>lmwBn3?*q2BqucDvHF*!LD%Hv(O6t@ zKygm{;QkpUQ?|MKj~JnZ+_FiR5ejR*tnRymbH;8Hg2QA>&Wsm+g|jRQB`;RwNO{k* z=t}C-9&yt)5R3A#3&FI9-sOyiU`*{e$a(Y-VZK3wsh;HtfrKDAp0l-awa3RsG-dIU zKjG2&>oZOV)zwLgWY{S`#}F-#Z5K9=XZJ+kuJOp`4X^RYqSLS5Lj>O}2kwSTY{Xg3 zEhICHG9Hdfm@l)bQ;Xz;O5+zG2{=)SO|Srr#K9?Dh^1c-X9TVUC|No;a&9s1Gq8U+r*@ugfQSVq>mBqH%!Bu5(Z4W+(isfowMT6JZ>2}INdUB)h$)MZn^Zbs>yS-tcc=t) zAwl(x{U|ZASUox3lK;_!8g`SlJ#DGTGT}KKk&enN*E<~aWYix!3P+3ds&|oGRZ2EI ziB%t=VqR6yRQc1Y!Y5BwRd`>XRFyApm6T;y#(R3gR#8e~GQB~0${0{MNofY-;V1G{ z19DBWOSb#-eoFX1T?xO-;dc%A&fxHJXHe{gE@dq=wplLmwKP(DWK{es8$|h|XJh#< zlIVL7yRE-V@_jpwzyH3r8IwB7qyPg|`OAV3w#b3K1pq&>`fo*bml=aB4nkjotzl5cRUudUr7?BiCY%cBTE~!Wy`?u{EGxCfe;qkts$3of`;`>A5ReXiqhLaDh9Vwj&`N9;6tQ*vmohwI z!=n|x*&+e$GGL0%Rcs5bKFw=;xNC$wRT^e#9M6NJ2{p4s} z8#@xji3AZZVMYsV`(1M;fLJbl*hJUhrL`H^Z7qs-MMgP9t`j{+JtyfWuvTKl+6wTu zU{I*zzwonQ5$jxzg{;Xy^SC-kOgNOt3s9rPOlb*=l~ZL?r_ZjN48=}P=3H+e*$h7x ziU%oSL5;XxNP#RKApsja#)f>j6}MMgRo-*$C>OE9L2!T&mY>d)+&ML^--PFr@M#g& zAut3o17MCuK@Q|w#y=I?ZHD8yk=JTh?RFS!Xgi2=V%U7z18)SjwdsYLTiKBxqWE2o z^Y3vym*INtv(24R&r%XXO=ku1q5g2ImM0?#4zx0?>K2+0mX!%r(?-v7gZDDvEDze& z5!%Tv7+`hSs<2;m_slg}vJ^`~(TlsSYlI=LOQmt=OQX)4cvXMx3OC>7$I=?ajKo;) zR(r_mN2j<^8N4H>KPzuj+x+FhZdf~Cmv^7~ds4e&?RY<{?pOy}J%06vgo|M7W1*zs z)y{?L!IPn>aMO7mb!$HC8j9aC(Aq(0ilP<|Zob|9YIjodg8q!W-E-tmtDAcLPkn3I5$Uwo?yW79Oq6i?fKcr~^SWCrJknP*9mruG%+I*4EB>JF`_(%z&Q z8S@k%H;$E~p?4LMd+Q)(^tBN5{fAPeF$H@Iu@Oh7`*1T)5pybz8oRB9H&voLm|0 zog|Qlp4F>t14rNiH(G$vq14OMC|Sr;RfX@-=53(OQ0QnK1Q8m6X-r0d6Fa({=;lEZ z%2%v##KtU>*&AS+aKtD<0UG0kW-oCb# zU`8l}Bg2vQj0Q=_mK4`wJ;`B=*);ah8}8&VAzpot5r0Y;Fwh&R++qFcgJU_{t0c?& zAsg6!9!+Bg)s9@c$Y*6E2N3*22cNuSr<=8}Q~j5M z4u6*glOPFD8051N0iG644z_RD>1RurY?e|Op}W0&sX)0GAes1ABqK!sOQkfl?;kSx zp|~R2sS9U(=y32|WZ>yD z4CI4A=jkLD-E{g6xw>CQrVaTVhtXnwgx>Jf)#5`jL!e{=$%3W}%Gh=AYd*sw_CloZ zRx)VDmOKa&9TurOBXg{lqQXb#{gkDRRm0;cspTuJ%k(-&C3ScJ_paej#2>-3^1OKm z`(Jx{Z6-5p-E}huqL>;D`plAu2Ey@FMVm5xPfdqE9wxz}`guq;Q~2uF_7M4^WaA3#9`Nf zJdYsy%#CcC5;V$iaETV_Gz&tUJ5EaI1R4v%t4Q6`uOfVHU(|S3Hy&;q9NTo<{Dhz= z8IyS$(8x%)Q5+03B@vhFa)gWqxRox94ac%qx30ydK}LWKv}tR`e^5btLzGi+Xtqcd z=X!jLUd6EB2|eMSG>MWB@ykBXb+@a7MA8|)(F+wKpY*<(=d}0Lt_rbWYKBC-;=TQ6 z&*6p-8x&zW4s~D+9f1<&Mp_*cNNR6}12b{5?hO7>vlyn|$moKTHOR}yFwi%D9jqY_ z#S(We1>CMn$60oqQM43+#<_rC0vHDu6L2>{rN-P0qb^S*&>iE9iWh+i1!q*iX$b{h z7?3XUjYMb}NZm_(V_d{w+Kc0+BY!}Fd5{_yh)5U3a2XIPgG1cz{ripqjUV+Fu@1GN zB#9QIbdQf^_RSt#v#`RDa6$?W`H1Lhcg%nSG6f%$tE(5t3IbDm$MTVyl zNj`eMklgn+$i_3@dnwVlpf_ogcK<~ZyyF>A=xLY{F?ogz$)dV4#04YS8BQ^cc%TUG zud)k)4%s0m^PenoY!G)tl_2;8MoqauSxAo<0Crg97(*mPPzr(1&9BojHpm-~oFtSe z7_ojyB34RHKgy1qpV`H~xP{R)&GM*xpHHwbDK|QX9?@h-k3~+v{ z*AkEH`WttE6@LvoCR@*H9HGvK4*>7>*7_)tkqid|vwfU|W)r0g74>?(ElVJ%xk)bz z$(ckM@_}~Tk%h5)!8YcnCpwgqms~y{H~yWUE&)ZUdqS{rEN?2@nTB)1=BA4DvcPglFK$y zR`Z_sEK%wLbSAPV@bspq>Oqcc zUY-rW6BvOf&iGaJ(TlalbICF#}bYE4$=$u(s%OXT&bM+}Ro27&vv)o#DC478c)VQvlbB z?h_m{p4qQoAD$z}bGFDIJTh)ExROI$MNGnS`H;D`Sp_Fhpq33CS9~oYS%y$zX<<>r zW{UzOpX5j)R4+hU<3Rw!>Rtv+lI$DzyP?S?#8^u!?t~^cBgWP!NQ~})DBC;2$)#ccThgS+>OgAjHz1mHK&WMl2-)GJy9Bl2H7fq8W)pu%GF2V=5I+ zJfOl1kK5yn$$U-{9hXss2eVBYLL;tWLK2h_Liuimt}dbwG;08I8iij zspcNY?Ez4csTRp`{#`~E=@B4hzW#wEZwaBo-qZN-E@zy@P#2$gK;(GL_V#H;K+LAJ zB|H}JTOLzQMB*_NhY^8kfb{}0TU-zQAQHWp7h+?)>AcK_+gqIWhl6LM`-_H}9xiJy= zTWe=?(sIhhDwRw z^VN&y9ws85$*3sFjN!@^A}25#?7@< z$PyV(FiON?)B*QM5rEn6M?li?`zVJ981mSNCWcYit`A60Pw z!3_-}ZNGgabBZP*&V1O&a|TQ&IgOTiBO)s7gW;4J-6#^kt-Ivh?pn=7++2!_P3#!S zJ-!rszA>?Tj9~9m3$WZb0!T#qG)l!U!x*9eDBm~ghdAu*9rOm> zAWkDAZ?;OJtnN5b@|y^@ZLnJEBsU^d&#_>cxNeS>=4r8l1)br9ij=w{WiuFopISz^ zie9Sl*52&Si!=Ff^PK_Tt z>^!>;{cp3#*!a;00Q}z`TW#tapHI~J@IeXaQHRHMyAT}F1vdiEZaS!kZf`$)P@?Wi z7uNSTJpkU*YXsfi9<{DcgDH~oa3qA`*5Ific5?3X$Feh#G8O?TBb1Cue1y(loT~Sh zxrWgUggrXtq2cZMC^I!FseBb={c$j!;Sw+&o0t;L54qgj(ET8Ix}lK|_aupE!;KbM zMN2nZ2zR=i%(3GLgtk+h%_DAsx{JanLo~y92a!ZYk&>oFBHp$N?&R3E+o;$X8A%4| zn@rQ?l8COUD0+lCY%O%J@yPHc%SI7p1nb+gW1y1fvy3dJv$Hq3d4^xlFu1nZJgIU9 zRe64p2vh(UF%cCvkWBzDp@vC_?m?Zx_nve#@x_%#xHs*I7?6mPOPp|Myr)5VFI*|p zkKC67e@_yU2?qv9x6ET_W(zFcJus{jej75JN@lF0P~So>A`23nri{n2qi3j2TfqTD zgS-5?s+%>NV1=Luyr-myeq|5OqdCrLFi%ZojdCf%$%JHM%o~(J$3^idG2CF0CHx`& z7Ft@v}*_wZum!%#5c-m zRe7zkBh5{hrdDKg1XO$MgBw>z;g;kyh+}ORrt(&aZKNG>B{(w!fw3OShSza+e8nd^ zR0Uz_$obC|>tJ-9%XG*Dkt~`J=n6vtQZd?rD0hY;ZlejVI^ zx0ZLf#f?G29btx>3;1O*20nbk-mcmi5A6U`uiisM{_xw3%*8;Vu8KcsT0t8@$FQix zS1E54G?EiJn9fjY$;(e9yUh72j3y=cntP*%-Y7xoh|rlGiH?V26mNJaPAfTH5M_D| z@q0SEF0n)7S6u8lyX*c63sl$jm zH>C9yGgGm+m!BFfr^9zBZ{Yn@1IE|6f1HbvCSQ~3w{XY`l|ru1T^0A5jqBfD3nx-8 z8I8$wiY;k8oKn(zRQb$aXAxx^(IR|8zb}pELvg!cOaY~g4U{}k4;{zgQAwjK9N%zt zfoem55d$8eoQ7mU_4j-+)_xN?>WBEMFUn8?zTvz_;4_)cRf;f^Y(2jm;T<7h)PjalQ+-J#NNAGS1OaOpVp&_FndCWJfS^sH00Si?*VD97KI3sWqBrSo9& zb!v~U(T+rc4Qz@VtXQE2RM$E~`Ucr=gxZg@18lXJouUNnJpr8{R!TpkPrNUE5np}* zLP5Ts^N3v|#17EZHJVs00I9v^6G=_pi7`z9yZ?wY+Mb`Yvb~`v$Db|Kou101q29?b z;YyL>qepi%8D?X0GJtg~n-ICfo6Up?Ue3gB?NsctLm8*kkgAE05WG^9!;ai#ZneQf(pq-O1isJETXHyZk8%fRKE3vOI5%^*iuIt_;B_T%J(=)x4hRfz z2?a0==QP7+LXDKrTq=&q^^aFLi-RF1go}2Hvn7WwHu%O|nasRg&LQW*E&#PM_o`!IXVB?^ zApM|(pWiuUO$pl_ZvPU^!L*5+9z_$&qDUm+k-V&sOtOZ55UuI}Awn1nbIN$hFv)rH zd~=X0;#gb>4Cu}!g;hF^qaYW?JgggYz7+Qv@8vj`L>z2Tt8{xNKI z6$+MhZ5IW@UXZ3eb`_89FC+7d5(8!sRi|hjK=}uU1l^BVMOP76n8~~TVT6BQ1TiC7PX_~a za&8X~CA@i*EGW+9ImAcrjCbj!SX1_?WP})5#5n(!WP%#L1qWLoCO}GpGy)AQqcEt` zLE?iG@%3+7xwDG^+Q zOeb!oxAW{at&L-+0$fgI9_%x327n$qCsVF94oLV zEGJqOjIY5aunjdI#a!h58yWF?g%}2~FaHmNS^jhPKT?v>KPyf71o&Af5an)$N>e{2 zR6ceP$w#ckqQ*s!9?gOzpU7Zq1s^dEUWjPSs{-yiuSml$Dp0ecx6SG2;_JF#*4OK1i4sj{qkgx)~Z zWyS%kQtG4pgMGFb6=^aZN8^Yp-Hm3?1Ybe~K1F~KjSKqY1R@fdCFPP-frX9PA1O-$ zx1YF>n6j|dWsxU(K`GNDJ{}9o!+e4cC6BHIH>(YtR42N^4^^&Zqp{4CF4@hB;F!%? zP3`tH5>Bh2#1V%W=M@cO2t#JZ^M%O}4b%`2ThrsusHvM%8QoP5LE%qjT(N{$&<8wH z8Qu3~P+tHG+0fH~jW&^YZ19*gA|N}UQf@*dOE|E&FPUivBm3@yWqF=yNawH59DYi` za|Y9xW*>B&xm3ay@J~gD8GskZz>r%7%pYhRxYOZ4;h`KLs00{FQ(OlTXS4+7*=3Yk zkgjmX|CI5dy$~bF?w?^+{qL-XOFaB2U*ek@R4?&O8K&Hv9gZFusuhsEflG=`oau$j zjo}Nn_HSTOIlg9;0L6wYpHxb4E>5n5U5LFY*hC&?J1Wb2c^Bd40A1s@qR8Xi>`#<9 zZbIHz6~ayE2|FY&wz*+tj_F73Tk|m`;YUUJKQfN(V=jU}<(TgD{$0Ccau)Bkj&8x~ zU0nQP1cR#`YiSl|?&D87s#(c+7CP&O@<>VO3+GYguJu3Ly{~p2tG9*d9#*KzItMNp zEyUuNjYK~tOGw)H3qWOFQ zc4>$tW1Y?@NJ1&A0|g5|m|p%xY#W*0TI4|Dk-oReHjoi;-}p1i^99cP2Z#I5zm}7c zlJKQM{x5($pAk=Jo_UlUOZCwa8cG4(yCl$W`Kb{l(+o?G!s1e9o9|nanMD21-=JY{ z_%%p-#WhY0G7N+O*cG}Uwq~!QK-df7IFtpH->PWE6w}+2J&MV+zc<($^alt1!CpU_ zry02dh9&P((xo%pm8}M8)Qys^2GC`=TFZbH2N&BenRJ6hR^5`95*s_=RJY_u*7Jf{Oc*6mm<>u8@^gvh| zcM(Z2zbuQ%()qJY$VMg!6Cqa7=2vx=iFWIPK{~%lG1?9>#n~K=^BK8TLjp}1f&su2 zhdYWLxYGqP!j$5y%aG_o%WSDIhLXRcH>rw{CVO&(P8jQ;4zn}P$BnhZsvr??sriYy z_8y6Z{>4->lJdB|1?EiA*bV2vVDTkvOmvqA9*Mg&)?q98vQt0pH7BCW$wlWqam_x7 zL&G^kj=USl**|c;aE-vpscmjgvf^2IriFbmM5;!_%GScgZo|Rz!Je8D;sj#wdn+n= znjBn1pk~246(LSVI1!R?i+LSF@ew!ZfEK|=2vF{^^~8w4lOITk8IUJtbBTC)g!Tu| z2TI}cTJV1-r$|Jv#JLu&IYz<4b6-j?qg)#l=xcQK0jmRu_oei>|43%9=|-u#HF)lD z;B5$&xGAn$Ble{-Gb26AmQGIf%8p48{w0WmWUQ8m`}zzn)NjF`&SH9BVjhYmnpgsM zVZ;Z>0u@vZvn$SI5{-Feix77~AB{*R7isr83bi!PxuQ`9&zuBY6cijy7fnP-DGxizFiZNw7^&vqXY(Y2*a zx5sb6RBJ7*jAg==$*$wQ2x}p6l`z4qQLPfYhf8P4mD6uj+svA_L95g>;~cEtg0t56 zuf$?arq(iyC9APp-cWKKC>HDzTO!-sCUG6u7A*iZx^VKn%i<&S?(J(H`9*m7Ls~q= zh$#V_6sFp)!3v78|D=kfM!V8OFH{guaUlPy^dG;PnUQpKfheaDLG$z1XEIeNZ^~$0 zt5B9v%5bNf+IHE=J`r@zqS+&be(WqZYZ2%g8B`>4$#i(>vAVus7Cz&)Ek(2K>QhuXp^= zm1UzZk-gcGx&-#_s#-0Dy#uWbsk)r(K3KZTYOqR@v%`DoxyGF^I)Xv#f|#?zmWBPQ zyC+itSa>QbUzO-lq1*LQZ$*fExY4$u*nn*fS%5{#@8D8|gVlf(%O?^W30T^? zx`07XuxiCr>gUlMt3>;@WM>v|0o`%#}q>4MLRR=^_r; zl}By{+1@_~-RHmj`}REln17sqob90Pc@)Qyv;~*{3qx=;Ba2L$gDZxU^3(qa?H$aF zSXx4&Bb`;o1dPET9|~ssNCe)tL~82&iBINpCWAxf(&=AhlVcMxBiAZ=r0Ezd-5boY zEGAO_h(=L2;%Jj- zqaFY?(MC%nje4St`XY>)(M8379we3sGAhWV6Kd2BFY4=Vbs*8kHXDzuoD5?5`Nqh? z;mG&^E+-d4uy%y5_Ka@`5V^2}jmnii3i4=t!S0dn7X-nq_^jk)jxX5HLYXwk^J|iY zq73_KmB|^Y7~6R=kin=}8$b<|4Z_WoAwSS)j>!U4TE;Q{>h$fa7suzXUiRdX{_gE- zVPph3{IJmMn7ft>%}O_!cd@%7Mk^QvyF3xFGl9tcU^(D~Wn=rv2i7#HtJ>X*EH2sF z&O&*Y#Ya{KX{8`~_x81Md?4ri&;njXi@^0%dq*dj(cT~fnUPEhfyN}u7@x<+-B&i6 zI1T1PlagJ@kW~e>elu&=a{K4J#2xRmHXcc2BuNEQK%DA<1K#;wCkV^mv4U2f>;)LPM3(Vt~F`>>#TJ^0ZkLyt#KDwbdjK%EAAt(LTNpTM z`9_=`)yUo$EINpX1i29ja=AL$Zqa{&csm0XCF}R%^$9x`fM@S{8B40qyrIol>W(k7 zX^o0^yjjIQH5$C5{b_b-U7R0}X&0nX)&A^0WDf4#+t(|wYeH_WCy1K-2^RyUL6pJr zHV*I}I#y;DR!Se?1&e;=V}AjgSZ4kCRkJia{Av(}DkC*i6MI59p-YfLrjnBINv@OR z7{2B*Xr7R!awe1SAdh|Ji5+Y?h-0BBQcehPvd56)ONJqlQSf9a5uH&9f=o_GHpeRb z77!qA$tywII);F)e{VdUT;|EKvK$E-l6&oPluhcF0i3q#n%nA5;Mc{L#dCoSq#j4Z@wjOLM8Um`iysi7HbgBzo>3yQm{p}sBB zlVTt(lq$-AcxcNbbTAmai7aQTAfZt=n1FQd2EwzZ8OTsMr!K}-c;2Tm5A9;*RpWwD zmAI9W6EUC7=%k9`wmQ943)O&Qln(|^d>CAlDH?4%o|9HGt+M9ePk zH0V5E{U}c}9Kb~&ry1+U5`%F~vdfqRVcusXi~ZA;@Vgv-*MRQ~3@>*E#a`%A)-cV3 z_h*;*T6#P^GAhFMl|M=;077t6d6(n?jq*vvJ*8ff9|)CWrV0XITGX7D>P`^~(p36l zbUE`Shwve$XhdIK4SS~~baAr;h%34v%6`Es{GM8K{&}Qrd<~u_ufa259$X4O(Y3nA znRZb@F(K(F7+>zH<2IrwB$$hFM^O@@U>dN)LGqmFW< z@;pk7$cu(~C%xiQPzrO% zqDBe4zZ)5?S_2h{uuMtUmzP0moheIw>8MPBh0UX>d{ivv0+$F2}stl z%61BJEeJ5n9lwOsh(p<4@Gy9$^|^vD3-TEbxy%UGF&bw$WQ#e7^PMfk;R#8c>{(=Z zx`6OQ2~ie?+DC@stn{ZwBE`M<0pj2Vo$+j*{G6E(a`ZSzv*F_(zx{6iA2*})@%Zxf z;M?zC9$%7w{MSkR{qx@VCO;pj8f^WL1mifzqUOst|NW1>|33Kr@zb->-%chsi`Rb( zd++C@e>#}&jeFpCFDzn}6ei6`xB&aL7zYWch=e^Im;!PubHQciDBzPYkESva7*@7< zRx}Nn|0AJd!OE2m);JkoyF6Djzr+Da6d6CjZ<{2rh9lTkkTgk9z((2zppZoCl4H&t^rjBD;LiNS^Rdo2U2)GvBU=dl4nmzD(_y_kZ~D z{MA{HzNf3TF&GR6PY({@|G{A3{y#X}fBL84!P94Z!{J~!c>1To@af@T?@ws3u1;1x z3w8wjX|QrzZs$Ie=fel|xc6cfWV8q8(CsZF_h^Jgb_U3{)!?lWe;L{Z)8@m;re)YYaAiNE-jbm9J_C|dx&JR_ zIA%vtkEZboZB+07{bvVz`_BG`@Iz00}D?FXa=WE{kCH^Di@VFztPAg&(j zoW1+Yi|>!$o?rZM{N_~$b3-kD#A=p)M0IHC|yqSMo0)G=k0-cw(N z3&znm(4Lql`89*I9lAZIl7uIJ_mDAnax00E(r9dl(U_tRoXQ>fi=BLI0E3+B_I7qq zha=QcZFIz0+2Qz#U#RmUNP|%nM>K-c?3?*&$3boD5rWt*U~FjD^8Xg~^uO$5t7)($`=+`Z!0RXThMwv7!sYZWvR z@_(r9WGTEo`^zewi!p7|HKUk%KMS%DCpi3T$@gOa*1I2nBO^`#Jq(Dy?aTeV%Z1Qe z)`1IG0qxGzl?L=H?2R=er_|xgF5fld!&6Kwwc6H;NsMVDG-T^`1fQN1&7B5xrZIZV zO7kc*LQchwdTB{v9g#h@`bDAMcabV@ZXOGg%qWj^?-N4NHvYY^;VN)N9qE?em#_O* z!yry)!LYL*`%JFS9 zf8tK+c0qbkc~_ID^G31%syMR8r=s9TM9F^z^Vk|6xlBa8m&)52E)Ovwmx~m2f;7$h zSHlkK=mH(awX!=+m)|Tps%M=YV?}r%-`;LH`c%-Oih^5+$8#3UI`EeT7(EsAK!o7w zj5#|AR|z3o(I6e3wD{-NBlJ_0gycF$XNwd<#qoQGd@Ef%xz~3Xsx1Wie~`GuQL=c? z^VaNcu_7ZJlWXzXumXU<%~EZS#yV+&v_1n>3(2twkzX0Cbp&6*I>azN$ocY-<^|%` z8Ie8#<2q?a0Cm|5uLXh65~1`;YM|j=!p`>lk6~|k!1i7v+QZ;@?0ZOl#0RHHMN13$ zu9l`}y^?G3nMYUA{D|kwk77)3QvA()kwcR5BP8x`{jb>NL3*&58V*>EM#e{|TvtUfoaGx8?LBh(MqNn?Yk$U0!ST29mQ=w_T7Kciz!AdG>{t?k^ zk&Myd0RGfB{%*%Z*ZN;=y^t~GWE1SNdxO}z0pk<4y}7^A)=!uNhw{)XHd49L6oJ~) z2tW#B3U>Y?`+PKyRLqblN87^JS9nB$<}}6Qo!-{TM3MmxL};3GJ~bi;JGC8$!RDDE zFRbUK(;1W&8Bc5US*2Ygf^`@M3(d%EA?pq8>&0;n1R5Q$>*g|!MtM#@uf?C4Y{kOd1K;!K;g6Sbf1(_H6R*a`h zJ|uBA$j~u2#CXm&#@?})Q?WNVpi%gOt5oo}-fg^?=^ zVhKGnttXp~o*5ykNRN!YZvtO!|MNba%}7RZqR#NO_QcPyZ7Sw^5ew4PxqS%lv+n2y?x>0hkb$4Ql_Zg{s|ba^r@V?=(fRP9Zg6gI zJ01vWkkKxg95ppn5QITrzUwOKNrZ~()?GAd_=tXA5DGn=;Vhz37x%a@c1B5;5Q1Ga zml<%GhY)A@3e=rGVu732Hg6wQtaX{YMHn)+<(cfPE2Tn14<}^T?1@{4)~#wXQ9}$o z{9A9QMazj;svnB_U4=tO{Lxt!ZYeK(-xH?t)&C^Y)!)k$8Z=I%$*(8!-g&K@*ka#w zki$Eh^`#qavm{bGSHkk^z+fj^&wJbXUV+K>P4l{EaBY)gGK(utkHtJUoS=$2Yo8*E zFWzZ{!gABp7-E}=qQ&$QOU{$YZZ@2z@+vEweMKg_=jVynGAT^~mKr-j%mER{QHwCS z-6;=lK)75a6nR+YiPi!@y5TC6s~onv((4>PzTUWYo(2@l@aZ}Y zs=Ml%j!hA!amiI9B&^be?OCT-sl!7`$RX%honE%KcLarK$Ooj&LiCK~y{^fUarZ6H zro(3~!sTYb3*s*5fH5ARn3R?q_d5w?STvoAvihwjWEntfoE9EK2N^2f0zGLgIJ(q8 zOib$yT71?lt{saX`|mJS2@Q`=PmqMv5Mz<0upo$5vmPWN+O`otK0Og-^WOJCULvaA zLE8n}7BuV)pLYlK*G6j~*@aQgvqcBXMdZc`VP{->^hcNm!;X*m)|RfK-1t4k2{i4k zRn7;43y_@Wobui@)<4Kuj`A}|FKadWd+lua774pa)T<~;d%(?Da-F18YYWw@>p+J4 zq6C!=J0pRHP98FXwj36T&8cYF93GN^$yn1N(j&|U2uv_{&^BzMH4ijjJ!QM>>t%M? zP6dtO3ae#EOd5_>BQIRnRzXwthU=p#y0He^Srsnb%JwfkoIE^|O6@UTWDpJ3SjboH zM%n0{@ZW)>#PYw<-W1wv!}x4~uzfd-G9(UdPYvT+g0IbQ`{p-!1RCED?3HjCCE-bq zI#B`{=sLQN#XGnrTNc)>mVg)z|2lxqmhgc>$516#iYISV*&A}yXo?o-g0#jI(t*+u{Yk@6toWZItTV~Ct-`*qK zHA*@0aH6sYHn<9BiJZowid;JLrK7Cf(KKm;_&>_@>@Pb#XP6cezM7#7xlBy>G8X5+ z{@&vNAk}5Ac^^=?_l&fwY@b_MSU-I5I;8Q6M&?xZ!AhiBSyAUSW$(=lr}fpGJfjLg zA^&JSC^reHf>q28S^~_zIiC|W!!ZODN&^~+?EwuEI>VW``U+#!MUIi_%6YnoV;&$S z?+IxVj-Z;1pn=E)eGm~YRN-4R57HcMUw+LN=uM1wb`i1b2qQ_fJtrBKv9V@2o{I{- zEsn?0mPqzypu*nCAg8*~q4(|H))vXOYJs_6OaM(;Vh6Bc0s-EZK37?zI5JOk#_-ac z6#qB@CUzdA|KtPq3#3qW-m-2 zY>+k>d#nr-fyTAgeInDAl*=gU0Xd->97MRef%&_~e(MHqZ+rRGxPdCTQVS{_g^JC| z(JFgij!tpvKDA5!y*`!kA57($^`mza%$rAqsE+@zw|}_r#DCb^9~|s&;y>KSQ>6no zwwoEnNiUTiPxuVM2Cjx%JgSpv3A$y5bW((~&TPtS`6;Na+QW^lwoIyJ#*`9HRjq8l zEWP!X8RnqcN{*c;j2lfTI`CL-SsgZ_pK+AUaA+Q@uF}fICIx}x3DYWZFn&N%Q8;Av zdH2|iprOXV8+(IwT?wC?x7AZExT2aiKxi;+RX(2uygi;47fcHu8<_MBN5W>EkO_6s z?d?&^$_)A2+y6VC1$&22+g9d1>9|tphcXtQtXF(cAJhzY92WC#aO>EtDniPBb}6Kb z+WS^(DxU?d8&;M5Y@3-qv@v8ASDzmaW=Pn^FpOHFa|J`5_Q#&j7)Tsp!GaP9XL@Fp zIVYF6&1_bIsXe1yU^4m2yG)FM@q#a8nwA||Rj_)na?Q#>WM;8bXjM&p?O$Za-88to4jmP{<%{Zb3U)v;CbT&knd&J*a3< z_h?572n{k?feHGs&H+$nS9K#%D7dyEDWcGJY>K$8GDz2W#2dQCqxOrI)xN9MD)U~J z41@UG9_R9w?y4tm0y>`I?2Kjs#nYRz9y!I<(KEN@_4H5&>JTTCN`l`OV-0Cdm|1-D zmXmw9vl{=s1U^o_7Im81NnDmVRV}FN89SM2nd8WREJ^kxC=Y2{4!4wqZ-lH^aC-Vh z`|&fO)s3AI{je6OnIo;~mX*ahO;9PwmL@(1yw>DH)~y>8BS2ldF%b#=gsqXA8c!xC zS|aNjLR~wpRr4D_X=*#9OsTwyt&mP^RmkKfpDy|Uam0$wKP(8EydJfXk#C&pi%JTLtfqPj?O?y&l zB~!HFsn=;1&2yBs%-VziGFzCubIUwj%iOsoto0ygCk7LvXeSG6jidu}J%|?ajtRD^ zn2zkjhZP_$&BmyxfTvgur1Nr#`;h~6JwqvT#&NZo$e!04Hwk5(IEgfU24W%u-;&q~ zcvS*em4vRv=_+#EwI&2yi*OuX;fzK(J`O|h+~HKg|FMmXupIo-q>F?64suVPBot>? zQYq;D)E?U+=oN8g8|9qmn70r$-nYln0lF%Ri+q+Y;@CT&?#CsN^?+X%;|a;e_!Vd% zUvTJeNGM3J|62=u%YiS8?mSA~;vl>^!()uvK4Y=!NXd$kRIPIjct?LQ%~E?4)oGykm1|2I6?bMpV~?LFIjwz2=bkEhc9 zQ>S(9fhbMd;Xmff;0IxYE`xHJmAn5Y(sQ_*YY%{|I*&69qs>9J^g<=cv_(UgTuoO{lAZ=&ak?X zHJ#;`(?yR}_=K`OD=Vsk+D5!{GEZYX#|fu!kb`a#jkWiaFnV$i9})GoT^u?SDc!;- z7+>NfJhL0B?1+ifM{l^w6_ML<`+rJ!dc!7YLD?EJe9M+h9rUJ}^ynMJ(>E@os8#DC zZ?q_K16>OVg;|hBCnC@VgSc9J3=LKkI&;Etv}~dn&BJcWWN;ctTzw%q*r0O8NCY(T zb4J#7xnpKW;aSP($RGr@N@owa9ls{wPFS8MJ&lc6%9ZS!Q=vizxGd+yoqBYRT62HY zn(M2r!T)z;#_rp4i9j_RYpQI^{Y8xPqlQQtw%Z1y$llhH7NaUcUBrI!HIl{x!ef&; zmCV%bWnql`3`8t6E0X4qxcEZB0D2L<}nx0cPbuBY|=KgF4F znrkcxsJZ_iK7BTD?*9kFy@SpD|303T?*EdS;q0wg5>3Pfp5ko<2L=eA2ti-vc}(^) zXvIsbbO&X$kec%D`a8G(0xb~c6`FVJ3%utlLH_F~FA?K67LPo3Z)@EGbz1qR8&N#x zr>Nk|qFwYD256DcD3)sr&dDyaui^;qpR3;S6?bP2mcA}+@l6I;$v1T~(xd6XRU$j|bqH{X=&e44>3 z$T?e~7QBC#1P*f6yO!vA;8_UfmX8dLJnd#A z#_bJ&>gfNo!GUZ4Gu(f=q5t>sG^PLBhMTk<>nY7sUCM+bie;lxK`_wCYy7m>|7BS^A9pj z$2mlVHWYl(88AYJ^#N2&Op7kAU9)^T{H|@KV-HjE#+z~B~fm;55u;<2qK6tvJ|M&8I(e%GD(bosXU-;_Y90R4t0EzKiJYi@^ z0$MS+TL>A4@S27)UyOdkW0`S6dWV=nOks?ci#J{ocy%4NlMl_;pcVaJ!uJ19_4I%5 zaL=Xx!{M_H{lAyz4@3X$3Ro^f0TaeuoyyX%<~YF`nY^2u*@G zXXnBrkpg#2GAtq)AbOMHyk~}7;Od1ZDv;(A`3BEZdh;^M)QdDD?1;djdc&pj>Ssu< zlj|T0k55l@CAL;_iWB;GXz@iHM056aMsoTC&wr^=W-m`pU)V2BPhRR5*&;d4-{n|{ zkshH_oI#E+I>S1u38I9RG6A)Crj*`WD z*}(ZM%Ed}YL7qdr3D#IbLIy0uXpu+B6dkAOaW*Fz$^uA2MQ1^R&Rzkm^0pSK^vsFH z#mOxCW|31U&c_o}r%GNX=Qju;?;K?AXI-5#I6W2ZX&xm2P!Y z-^cU8%z<0M0PLXcgf#X2TIBfE`zWVTB0?~WgT4P1r%;n9#wg0!X&6U1MAtK%!2aa1 z%OaAnuQMFaQIzB~h+{yImEd^@g{~Ea1m_SgAG79+9O;gS`gVzndc8}YUl?z3DkS4$ zvmFz1xkypxYS=*?-tqbd%uB+KiTBP9XISh&lB0CtX0XTM{AmLj$;KlD0B z=n~_!(swKOdgIslZ+R-+ljr?9eCf-Ef~H1nmC7-LI4EWu7V9%R6O_r5pNyCgxEEiE zFTM^(sS-E(;Ee^O4hP&>4y$o9eCl9Vt(*o;7%W6xYqbqEX{B*!1-WK{^+Bu0RrqYV zq%Zer!~b#n_O-`m9qf85LS3DkL&a;9?L1y1JyB`Ij^{_a#*Dg>s4u~HZ%ACMLDWJ+){-_#0$FKZViJUZ3F06b>w&P})sIFI#tKNB!Oh?;$b^dMm|q~R!k zwQgS3ElD$I0Oi`375kKwge#McF-{nDUdXl5v8Vey3nr6j%q7_MgY2zVxWj-4f|($G zsd%quBK8J7{DFjE?XJSfC*nnYuVhJ&HJ@-3s0EQ4cJ^G((5z`)R<)NAO2#BULgz0| zjWNtHTQ)fBUgOc0Vb|_jxO_A0?G36dT*O&(s@j=Rn)-qh@ScA?_e}!x~7+n?JU=9Wmz{AIS=!HY$f+_Ue+%h50oNpsM|R%kj{%Z!U?F2)g$C)O=#w3xAGZ@*hLoyxzKa>-tu2P5v$h;_Gk-Vc_?B2Si=s;+L4v7 z1)-=mMN16;PvFIds6cZ6cUf!*XyA8aR9pbdqmqwjSkDUo)m+SPRbC#65e_n(0gLl( zq(gf+uN$LP0&aQyXq=nY1`u~bNG^~`!&-sT^yg^_hCFslhf$m?nx8EuljuF_^h?NF zF?^U6^SVFBG>gW0N5*i6qp)iFWR#+H< zOI1dwMc~{xN6>Wi&x?zI=h(Wq_(h7GKp+Vvj-=k?A_^7dPUBz7THsxT<5N`h;qQYdOr-a201y`2p4nOFy*LFm#WhHpg5mUpC^eIrqIBDU(bNmd(?>{k6Nh2D5} zzXYF#&C~p8E&p48{OA2=Pu=)02gBh;{&z3WI^=&90iD+yf0@ImGtF_LDBoWU;=&GD zUku`IqaegiR~d1)ttZhRHkk0b!U$U{u|b4iBfM~V)dn$!eery4fQ}6!x9>3R5bMi- zD2r=s*Q_&rt2VSl$ZKrZx{acVU1=A5eM+#~q~FL6M5m>Ob`FSA)4CE|l#!k6koLz~ zPRMI@5&!P*V!_Hz>{`>-&Z%fy*O~1ULJOxQAksp0$HvaZwP01}DhyD&Kz!>m60-t2 z=QN;rvWRm`9XwJJc8j=l>j_b5qb`}%22{Rjc1)&9xcN%{55H%VhO@jsCK>LANxqa9 zNS*!9K_ULf{&0U2|MgxT5C6yJ`el+ASU>q**%3S6_42pXx%s_1cQ7o&Bwt-+s{tYO z8yQJiuXww>4%cFjpem0lpp>r0sr>^<@RoHOyaOtjCQNK(-3f3mDbqn#y_A8efwQ}y{~Sjwqmj@R-k z@at+DgY7CPjO-Nav8yuKp*AK})|zc$M%6JkQm272R0A%k+1yA4ZQ2U?cJwgI%ldf4bTi%&!TABb^~Mxf(p>kLrK|L?p0Urz@I8~M+@JRbg^&Gm5_ zoxx0hj_iGqMtqsx34<>TG9EN@?~Z`Q-rMv0@RHmg*dOUx@%(RO4pw*mKN}X#|KZ_abN=7Qv*z>vVcUj%5*|R` zhwhUJSkjhki357o<9W5i`HMc98(86e5#82(+9{FgR5PfyAimNdqZ*NF^TUR9YmE_& z7;ROA!)u)=t-h6;!ixObdd~l9Jz&lG|8#$_@0|b74i7i_zk7M^=KTMRZ3D}9?{FH1*@D%1)s>{9SN%hLE+X)MvI?Y6eKO9EZHW$wm)qBZu- z?I&s{cm+F(RKQo4~+$*?ftm#QWUzZLwHTyB?L)zpv9EDSwe)= z2HHk;^f}3n%4gQILJC5oHY>%yMhadxUTR|8PQDCc(ySLP;;UE5@Rke4(0bjhAZ6SH z259`Wx&L804x4%W*iW7OZ?72t@#)^P&He8_o|60DD>@Fn7eMjhL3{bIb-Jc&lmKOj zZP*(T1>>Q3?>KX^ZigKJU*n_W74exBi#0Y*ClBFiP5+lr0M*g|!$Vj8x4-vnlmGi( zo~7vjXW->=7iWN@BZBvzT1nKJR=1_pE!8jMUuDsO*KhXmFq0$ zYnLof_}T&E^)@E$jdxTl`|PG5n%Iu}U~xlyT}>M1H$cO!T82VBAe>PL3-P8I9);@5 z4#p+Eu>pw}jnF7m);u`>-5`*ef0OzHnXRBM^)<7Mt$i~%8x`s2q$2gr=?c28I&d3h z*XF5s+T8zv16V=XvFi8&s_xuy}m6R0P`X)oTJv%D431(#ztrRR=Cj|63>kYv}*M zv;C(I{XZNYZtOqq3OYdVgg;~@pcmnFXaRkImv9>M;@3hC zSY7|#6@mH?e5jhhT7IbMBsJCrHmZG(%0T}JG}Q)H)~-|s)x`Of zbiCLvqta;zXrt5loOC+gNnJsyQwwixTAeohKii+-B5s=YM?H1&pQld?@gJT(+xY+7 z%TvPtpUJ8Avi9cJ2dxKCD|StcvI)->2Iy}OL?_^zrFH8CbQ?9gfe$juzp7D@9bnj~ zW-%|obFVg{LGVi5jmpsYmAKd8SUGGW^8fv-MwX-h?Q{S&^ndT*(2f6gu)n{N|J=*7 zBK`ji^Z<8z1~~eFyFCO}&O`_M6o#oT_%gnw~a=-Dj5-oK!#g^;! zG!=DPr&+N#?q{p9S?{yb`QK0nRCE3xK70Di)&D%(Ws=q zqp9Ag7DAmXS(CgY_55>5@|9BTKBJ^7^_}-7Td7!X_mgC$f<(1)-fIQNR{brj2rO-K z8D(G#U>k+t=cEuUo!AwWf{kFdRSdSe|IvVA8AP*x6~s{(&;~-_n)~12aKCW>+k3XR zx&PhA<6FCInt32z>z*cxCWNAGb{#v(Q71}Z8Fb14*lV>czi&kqed+n}3TIgqVhLjs zL@_@d+f}~~a+KjYxx!%&odWNP7CA=6MeeB_B{{`G*z1&UuFuJhrN#b_Ffs69^m08LBEc6J_yP%Fmt zPNZ253(iJHeKkZn4_3mZ$E6_JVtrY^P^-JDRBJh@RIxKfc~s+3pfniI-iVdS>)P** z+Kq}MknwN?9xM!FQ8TQ2E>G){YkD)_jb$|r>ed^>dVn+*t2C&)6asuHokw#_7F0_; zjF(MN@j`L%xeLKbrBq^VaO_B?>$G8oytKc?Hsi0vgqIru|KKgv!V=gAz+(>V!=}_G_*0MR=k(OD%=w!ji6%Jb0pq&eZh<-2L65~Q-|V^1 z`l;Oi(=3<-NzfmIZfzc<-7v^!BNAj`8*)$=|7|!Jy7^!BpKap5+{+`*f0EfQG-Bop zE;RCe$YV^sRLkL&%OS{;6OxVbOPt2!MwgZc_Txzu;~X7*gL>bxzk1)rWQ00;m34=` z;X(ha-fuZcI>wzxR4fZ3$Cbfvg7kHi)2)c&Ib3jLl+4FzKxexg$P1ECoKQeQLj~j% z8emO(^bN}J6u*DV?QoCd7caei4LJpE! zaDz15D3seznneknpw7SL-G9qFwIKnt7trG!7yXZjW{YHup6k81 z{Z2lmQ|?}`Z0{T;ba?Cn%A*jEgRE!Jl30ge;Wb;;5?(-A5@a{cD?%~U8Fo0|yN%~5 zZCib3$BkFwiGR?}Pl=tYUD0@ph&Ye+?fWTj+VskC7sKr>tX2_!MQlAx#oGOdDEZHv zB#o}{pPGPeo|T?<=YQU1tBOm0)|LNrI5^mI`TzZAPdEAB?&VqX{MS@bFNcDVz{>)t z^7V)JT3lfT62`(U_@s)Htx+d)9ArZ92q$0Nk!fJP({5?kX%(ciKq zK*u|?)XTQ|EY`5C!?<}3ehNE>=zX7|2f>2zsIIB|dJ%^95iSgDNWyCIwQ^3jIN>qBSfRQ+7O&tI-A5O+hU;xvxNLGBzgTE4MJXcRNnqzQ%21hk!UD%!R4gw#;e zMu$Hqr(Myw(|}3aQVWYZN@a9&oEhwafNy;7k`!kF#o^KO!SjKKSUp~nTGbFIP+FA| znI^QMUaoR~8*x=68tufD8)>i=YfYFc`RW*FbRHzZ)CidE?51O*jJ1z~;)gbTiPiS9 zOo|O;S&pR~%DUMXtEJT#7zisd(`h|!F6vM(b;0O z$Z))%{=SBtaw#afT?7AL%U0(Ikp7 znnYPnQHFyMO-LM*>nNF`I7%*$wq#d%H{HIC$$Y*@B6`!G;dtI1&w`BReVSq1p9fLW z7caa=QhyeVFSowxLt>c^GJDcFL;(_FCUo?T7PPzw(qI(D5sh%(`|DzaMWNnzkt%PR zKU$F7z^NkP$2Q3t3w*t5+#R%?#wU5nzkI5yvBK{`7pB&Jc?IYL9@ z(<} z2q$bKR1L#4i^5Z~!o!wAc5dRc~u;y**;w?@I4dRBNui%vG*aF7lQ?)?iC;=%xGoQc<0a&nbx#`i8(%z>BKwmf~!Ty;GDXVYjWBwr#W0&Pv<1 zZQHhO+h(P0+qO|@cGmt6_UZ01x-TLw;=70#BjSsgb3OB2D+(D)3R_AG+yK5Yx>b(? zZ#mgYz=ts_h6IYycFW+q>`)MVlWCBsT0;YEXl!@G~7lf0~my;{ckcS^Ag6&Ae?O1F1R- zqG~cb!?#NF>)t>uKF^1TCzX5%e$VzcD?h;ivR*C6NcMPtt0avyhoKj8_yB4O8}^(n z6Jza;eAVF^bq4dQ7juDKuk#{sehKhMf-j!x<^i!`7jif_;^aljP{ z$ihkrEfD)xEoEFFB)HpYWo;k)&e#ZQ@BG!D26TrV?S1}i=tViw3RC@;y|CqoO7?tg z81~L#D%65TTXLJF7G|~q&u%n_(uxT|B+kH~EW0MFW=(GPjGvMK=At$im;YUMeTM6gg>pdw&_R}>)Nc>lP@<+(Bvj-(qDm;{1b&?fTZG;CAWE) zxQ6FY;V_Z!+je-L*`8;nJ{v&>kH6G492~!!BE3YOWkn9WQ)i*Ym+AB}o4XUg{@x9N z26_|LhW1=~^{ps9s6;Jm+*DFEMn{ur;GAT-+c3_M6BU|entx5;t5XxM2@ax-iHPW< zZjMrT&2Udph^>cJd7V98jt-v^}Yjsd;mw zKWz3qV~gdx$2Xqp%2_=vY6`_oMrmy@uc)VDHwMkf%OxRTNf*@D^^V&M^nj?(^@*r7 z#iJ(st&T=WNN%TBLD={x+Fx@s=g*naAU2#nJ2(Vr8%<8L@w(z+^Hvr?h}3Vl0p?8Y97#H5oz5QJts z=lB{c#%`o2HOc_}8@YqdFHneK3-2+Ld-Ih=pya#Kg&_1zH1d-edIdd+y}6jjKB(xp zo*9FCYqKye>vPLv+di2&gUP4%DT}d*w^L@4V(h<0DwM)g)7XH2giz(x|WYq zQNw;U@X>`XbWncql}YKvyC3C+_~1Q2_4W#88ZK^9pQmG0;M0T()1f_YD~X1wt6CKS zE6dyQhp{8B)u;urW6JU6>R^&QYIOPANHde?bbkjk1B~btNfQP9m$+*$ESTTLe0VLt zuStyO(oF5|sdti7r*%Y_lssc~nBt+!WRE+{ntE_j>i%= z5Zg$3r|<8 zuHd=hY&<2Z`E0y)5|);~x<#|gFOkEfo0f`ZH%tRr-*dCPT^X72oCJlHufs# zl``=iCldACuil7Km;8$V1U3T~8Y=S@8rb(m_Wj<)W&DbN2xIHOW97iZHJJr(G84OS z8ih<7j_@zJ3gkey_DK;aA+cU&*Aw=Osr2} zA@I}hoq?!u=Qd45)eYS2S3Xl#{_hfX=4pA1BXgY1U4;0c+@E#fgd*Q$B!cq&$`x_N zFvXG7q`#*T#rHFFV$oZJFi3xKkNAD%=h%323|6ftnk!F!YH#nEVVwA>eweDA^71;zFn$DVo{2OAiAmNh~Lsy2WLF^ZOgQeK3QDOfh8hN&;=S*%ayjO&Er5YS&Fx2NL<>$2vN^FcYx1fMx4@ zoEaLZswVQBgf&Fgap?&mKtA~95U`qE1KY579~JCq>KMz%(4S->g0!PTWOYdnV^U>U z`BrO(1;wEjLz7t09kk8lD|4&_l6zxvHKYQx_EdJ7zF~2%Xe9WINsmKj3iC^X(+O}Q z;2CsteKnndlG7`tRFk*wB#7d_h|&I*IE}P~PZt@3yH#F{(y7${ZjYH6RyrmfOYpv0 zr4FAvuhbROybX+`%AF|3;!2S{;=HUOA7rw(8FidfcpF@L5j=6GaUl180ehiP!8U%Z z_Coh$Gf5g%CVU@WWOi?$*QLvcb11+imgeLYS$`#1bflIO(XnD<|U)yjKsk5RS z8_&vK?j`R+r6qCfkmP=r=5MT8e+tI9si4IZ#gYr{6`(KNI$RUU<*=^|w&rVlul_I$h1uWmUnFBo&D;_h`{MDIL-WZW4$+hPvEWE`@am^D1uIXVTE_#8O$}z|UBtEnjGA6uM&R7$N9rJUuI+{;dPQv?8;cH$JL@b?#42ch7zrQ-* zS4MU^yTHi3q~2h%ZyBk(U_&VE`U8=5ocxjTvTkD!gHo;AcZ29Wdt=!51d|~;Z$CVk z={{uhvD!}~z`?z~fWW!fcT+hq34?u#-DXJfBF+bi2KK;7i+~nHk^5t1XK@2Ba|}P7 zChlO^m;36w?febBAxAtg87f+x@4bmG4NEs{T+On%DFSKFtY9l|7wY@4`dafT?L1?i<+{ElpuW@X~FNQ+LhvXk} z^F@hZPYeK?gpG@6xhBPSDuP#C2o{OUS!PhFlaw-c7ztXF{b)d~C}(<88iXNSE)r3YAToQGm-MVwAC|tomENvgewPX3Hf^_F z^SH@$V*@-m0A!fT0eI0O0C}Nl$HL^ga-DVuG|wp#nqe$I03m#)5peLtp~>fpV%;j1 zY&|5I>o77ZNtSVk>phynRWfAEY8sSce@^F{s*BL(nexrtfu8;*1ACw*+`@jqDV%z^ z{Sw}MyY)L&QxoX+RL~LdVa0wc`Fo3ydV+^_7r!uOe~PYn&Sbi;Pv!lX69fQEGObVp z5YFZ(@!QV;k-^V^U4@f0#BjVaytO~I(+>Uqm8me1JjBaq^n}j5sm04jM}QYO>Gn$( zzlOKx`}OVb`^rk{j!(D8`{%|*YupxoBG16(S6|9T9zh3w!@DrL(X*fnuhH{_7-2_` zj0>-7oXcuUt1()q$Jyuiu?FSgv!D;};WISE%Bgy7`1cdrhVFW%H30jr5jgpKKd6k0 z*mJ52;{4~zoUqul@4ZRyZAf&EQ~%rPdmeSjQtKjOF;CtlbqcI>0JEJM0-8 zm4F2WAOR5wA5D862tX1*8Oct$KB!ukHYtF>P*5rWMc*2yE<9hN87Dv~V(-}t5IZh1 za8^p(HEh;)e*i!S9t#Y@Z^!vGCTd1DFFyQa}lc=T?$l$0h}< zVrRaR$w22_hAQCZMMiq~Lt1)xt6FzH7nJkqH1LmJjt6?;P}dmOzoL9VjDsP2qTN@a zJJ|@erD%Z&jfxwltIKV&pT_sG3iRK(sRQ~iFZx6VzMLHUHcv%I3{dsMaIZs&G30Uh zq`W~pfG&PuE&wAP{JbGJp*q-LZ;(vo3V$O@dI`NwjZx~ z(|CC@eA#{+fX*r05OStb9I*QhlmIdSS+RiZwE`xX0Aa&+ zO$gImQH$H6z-1$&B%*!_6Sby(3OyrHt8d)9i(G`eN^_una|#9!F*OC-0e4mr3h37Y zc1|Wv_k##Hup+>Hz(j@qljTj7B7s{DinifP0G*3^tO#6X^=o`+&kal}z$=bU`|pCW zL6S2jf{|)5uDm!~O#9H(fKxDl9g$an!;)f#OV_xjsD9V(0)E#87xDR==(+%GD^NT7 zQiaesQ!`*pIe;c0lMrc$^oC=BIpijX(}y7J=QeR~u@guZs!w>pF+!eE#B@8~Q9!&X z3;?8#K>(z}=t5MgNPqa56n>*8KtwZuLO@|42#^E~hx{|hLNZx>`oO~Z-#wdJY?gD&a1<(ivq<1Zx*#E9zKR7sPVi+19LPYrPk+R;->s&CuAb!8` zz045o$%c_m5hI6s9Px+@1965d>V+oKr=W$%i>DEZ|Z_>YORx3#m32h9VII z7~^#?b;)z49wIY=uRYNLQj5UWAX%;iQGjZVd4Z);OFv`|9J5N+1*3Q#AdmfHl45LF zzMz-F7i6#_HgNB-5ZS^Yzd?zX2PIZYjDk6-L6l@N{WT9%pzlx0BoFJ9?*7`ZRIcB6 z=FIWZ)%0VNq9@f&?kgWpg9C*6M$zxqRP>n9k1tPyP9m#T$k?$`26<~z z-fnz@50DvC?PmG~?~?r^ARz_*_s~R*{X&hsjnk3TIYrr*!ZadapDE@N zJmPAzxryjpISu){B#CFf_zjlr9u*HqRu@Z_AjDTfx9Nf^<5#CQQ8!J~9+mf=wzSu~ z5UOJ2C4hvgn<~@&uOZu^`XV$;6V*)%ZZy{`rA-$5*beSHYeA=HDYs`K8XsrM!jCh% z%F_GWy1W7mqbT6SVv50@m^1llCAJZ&Zz)utMgH0y-%uj^#A4V#_+s+lLcAkt#DRU~ z_8fQ@!)I+StXO+@e@vsAl%E!AVB(+}i5=fepR z?l|veU5_>NB`qzxWT*=?H&^DnM^Y%gLn|uI&1*-#D=RF%dbA%e4<9armpQpFEHWy} zOX(JE=txA*4Pp3pnJ&BJRDD$M4qSfbsSG%u!!sG=%dZmEJvMonI6Y2cSm`-+ z;__O_kz<7w@t5V=s>iHX*L1C##wl*bVe`s4a28nI%J`n`J0a#`AJEWUnt3Chzo^i7 zNzE@WY^nCHeh~tMz--1!KN+H3vcH>gPGn z{s8qhB_3f43r$P>7%f>8HT0onc=QV43V>yHky>agd;&>d>~3Qh<6(K5+;2ULF$QHd ze$J+3s7~P-!u!iPNnB*8Z{fO+9ZY9y4=!gK@t7G{%TdM>Niwp3mU6m}8WD_Zp=*F1 zTPBW4>cg32#vuQWoiR_I;IJni4i7VQQ(zcm4UEgN9pEG#r+xRedgtUIthLaVE4fX) z=bo)m`_Vc5L#yG|1E2C(5ENA-zFH~axO-`%H?;FKuXaP$he6w&E*Y{r2d@2vG2mGK zMgk8~ZP?}FiS{ManU_B7c@`XZ7GKC`M#tX=@4WS^&gxhF5hW`L#9zx zpEEP2JasHP+u-2f-t$$s5TwUjkQFnhoYSng=8W2S`-lt-1s}}Ptv6l3XOMPNu@Yi_%KT~AI%l;Up^GyuHuNAWh|4-5E6|_(I`!bnYjikM zU6$$CY{*{Jf?WsHJE}n&`XQu!j*82yo&h6SWwVU~x@feshIE-EI$DkPMux?*Xp!un zCI&YKc-!=ATqwWY@Gk(Bjb%>MZN5iTdxlF?`{x(Kyu_UmP=v1O^7MD!3fY}n`H{U1 zYm~dJ9!pjJ7g8YOUMm%HCcQr@?>)axY4=i;wmiZ=rJ)p25MIDEcsKwsMmBVY8svOw| zCu4Lz+bV0pX|c1XcT8nkIirdKGmE{I(b$P{h>%K%04w7sX?`q`N;#Z+RVxfwg0Y;6 zV7&vT*<-5zuG>UK-RRq1+>(A=b40U4`U|TPe#O`q{vFP5Wn&0q=ToDpxUut>s0_k{ z@M0!h8pt)&JLgwU%RzOOq(D@AMI;&ZmfK(Z6vlUnM01w%?}8|8C-_Zph7!{1R8XVjEsef2GMiz5cb`y%n)>WAAm37 z?yER{{+=bQcOk6$KWc;NAx!3!CLF3^5YvT-ci29HSod>BwD^-PM=VvXlTOED&~=hv zb1hM$in6N-#PXnQY{*R9oWwyq$!NU|OFv=lRt!>^y*5iXh~#nKq(Xo+*t$Z1*fL0n z_Ln4YFu*)!AseA)UM>+C;`&f!Kheov9OD{3=qir;5<;

PYc^5zkZ~ z`}ei5w#5c6<7<}R;lSzv0+MeVmMc15Mf!MuSWVK0;h_3jeTx^~wMrFezP3cSgw8Ch zEsnhseGoh~vZXCI&k`h>l!$M?{3$4YN2p;Za0C8XlQCe|-U6(Mqeu&p zgZvy6E^LbH*%(!S6`<+1+7{NKwP*4a;t7dFR);+kYphGYnd#-?(ACz-hUXa)H-ebr zfn!4JiEVU>5+7;5NA=Sm=-Qd@*LPPE{X^;<+(x}!eTZ0~mjh7(9QdBkA(y~a>!M$u z0Cj{l=3EU9Kf)awP0b>?84w{Ns?sNgSGuo}SWZSD06Dvn$HvxVh_1#ILn5k0aJr9j za^3x*g#k8`>tnV1gNkznE6xCVLTnnbdCYPL_BuG%$b|u(Qs+ z%(md>`C=O~uWRBe-fw#YcspA^pr%B9+ELr>Z4=0E_4@kFa z&PR)jy-%f?K3v)79Uy0pwH}1&(8_k~+0X%gnUop2ApClI&%jaO3SCp(45JBKqtd`l zQsC}g48ag9f3`7tOSQL5KSg=2bVGXG$f*5g*^iZR9P#NSvCuDo`_r(|)_539i z;>|5yrgiAcCj(4opFck~I3g1a-sLDN!WEv54bnlii47Z(_m&FbXP6DES14U9&n~FY zZU3(|LZ6QbUy`?&lqyTpY`lx(Vc;~&{{*y0an8`$E3Sk8i0Qb2Fy2sAQ(^f zJpf;(G&=%6eTLm8l;iI8xfg%M^AW2CQwB&EvK_%uj~v>Ok51jBJx}j=S(m3>OYj1) zqK#j@*})r4u)W7&G+cs@&4hyP>%en8qF4pql`g1-U2l6pWPpjh$T$caFl8i=G)M}- z8<8R#y^JMIU-Jf%23U_~>L8IkdP_x0>q5`~J+YK1WL4Qa9Jp%uhAp`zhKI{hfs8)nCn&(h-4GKDR%o+Snu?-l) z$XIy7avGrb+w9A0NCk|?^I&(J<-EPGu`V|_ZMtpj_%!^MUHCYR=2-#!d>!=p1N`%O z>Z3Zd=7GaN_CW#n&4YMr(&?Qj2qFYx8*ImHbRd9Ywp&AK$XjC;Wt>d}^-&l3bFddS zq?8>w?d3Xs)q&`D3PwnK@_B&WpmZqFjVkrl&{*=@9ulw=TNVB~XYd$=C_A&BpeZOZ zj&KQ4Vyv`KteN?ZInV?}DElcQmzhQFB9GVXQmL>O*FBz!^ZFr*y}`f zcCrAU>_f5am3#-5fC@CxQozTn!W1b3K3=KPG)n}mp2mF3wxH1*52+98XR(7~DQg@A zzcy+SrqQ+H9dkj5M>7#XY&!WPHg!P3 z1BpWblDJyb&{$k_>hV*t)K?xfJd>9wq?f~>Ajii+XHkqE{b!7TEHE3~) z!N@vmKEsvy*a$ada}ek_N=rN1{EEY(-ICk8N-;3K{3!mT!U1(~-CmPWOEes{_E?fl zmo?J6f^GKxuIo#NVs!6xSOs$ieKjSO=)Wj`DjgJuZ8$t1X0)sQ{@!*6=DbnWkz&Cl z3O}I#u%WLX*dxh6{zZR|1H9v01}|Xiyk$!_cuSH2V^;lHI)FqS1Yy~nx<(5x?62QK zpKF^QuFPeb)*2q8X%^!aPcvp;*;7Q8Qvh^u98&;k@EIRY!ntzHu_%Wa;Sx|123~_8#Di)@FdIJ2iSXKJ&{Yi|l zW=KA!+1MmJ8QY|IT& z*2$)Kvk+D2t$G@{KaxbrK}*S3y3f*7xLRw6!cF2gzlZW5*}{ALmHf+MYx^rRg6CbUbDVZ>PGE~Wp=kS|p2&y=2ALv>8qvK!yfy(Fql@Z$AJ{z_9U_l;!`n-d(xUGLX89@1|`z7ZDNwR+S+9wo&b)GZX=jMgypXwPD!3dnN{{{*^kr@S1N0$c|A4+&p}i-P6jwSCOzCQB9)EIv z8uJe25diImBB@p<25EYKIv87fAk(FW{((kb0YCzf^*@k$#T_ zf2~v5R=WO@n#BBnQ6I^&e~2j|_TJTVX3gq=5b;$H|1_4#MQ$_ONRzh%h>3W}{6zW^ z-5c_7;5t2S4s#v8KU;oWiM9N8m1(-{QHMeTKz_-8N*?AP0$6#uhBYNmBu|tf6u){ zR$^kjbsB+cE1qOI(jLLh@@*U6cT1=Syln@7=M417Msk1f{4@AJ@!T}zA9$X-s;of7 zfrhfWKmnzL&E2|17LF-y*!XWeKZjG8Ymk`Y*@K8f=mMkqyk)LGaBEvuBUNao{oF~v zn0zohQ>NEDjqRKKQ8QPWPs}DIni5xE9GzG9-!Pr=sFz##dMIJINqA>iWLPG@5q(=f zM*d51({Z@#&-P9+3d!dQ8{Y$}V@~40w1>64r?7#t#p7paWtaM%xK>pziA8UC;BK1}oLbc{1%H2FW&7Vt}i5DcSS0 zOzH|Eu0r|iJh5p)BFsYS@k9CL{@C;SGhA||#~e{Y5q;|mW={@VQ6JNo`0LcVV^@@Z z(3o56`O%9B35n6O4Bx*dipH zu3|+oCU|F9ezx_qNQn|hLfu^%Bt>CW;oqVY^Zk0~^;03nA&lXGE=EYd@fO}326P<& z&UH4(;V-S;-*R4j!g6Vsp>TwdHzbkozn$h}#^MU3tYpEZ%rsE6OZTvSBf(GY3>$_% zEhvxyd>^$ny{pl&o9TjNnpuvoS-+I{7usyHx>`e=f_)UNe;5OO6p?>?pOf>= z?;h?8Is!Imkm%jR^%))k5;xqoT4ay3>Ow5ek9h8p8Dmqo5dB3)0=`0zE2gh-8I-QS zF)pt=qHskPbV*`<+y!VD6UJhcoNDIe6`mdj)6m7g&;2-Y@T{Rx8ewkRAe|#;qsC%9c%P={f zrzf-fd3IWF+!?~%zV(zJF&NQDx5x~})+#rd^n1ofN5zTKa3s{8d9wngy|zUlH`mxb zza75p;9Ee=b~Pn*@qD!)ID;nQG*e*jP8>jyrrvex>>y2H+di&7kF|egiw;oL+nK#z zxeEByrY#M^ItzY}?Y=%8KI^?f|t_wq%;>g8Gev*LN5n@-avNIDF==M2Sfv zw!(JaYStI^)B4_B_B+G9+hsc8ohH{^h70s`l-xHY-x%D`{qaYB2(5#k*%;~CBBacz zU*^7*8xUb>-CYxrLK8_rN!jy}c4*l)q-L13m@dje;I4k4Xh4bCy`K=E%YRE+X&cux zhVNArAr74f$Q=R@O_VZ^E6aOTT=G3SAN*|WX;+wz4 z*0vbt(`T#Gx1mo=Un4^xFbF|IAl*PK<*hAN}#=cJ#r6gjw?C? z^uW9@!|pH@IRtxa*l4+U@-bBRW?2b9Am2jdCO!ien#lRsQ}TpPzNF--z3-GSZS5XU@NK>8Kqc=g z@Moge`^yh7tU{NmhUwAX3S??A=AXx7bvcojYXAOzB#nYZD zW3>g39jWU-c)=c9W3STE(QXPUHkBE)4G_sNaRgMD4GM#K22RqU{q!6nq>fbsjk+nI zyi^kNRQTZC`relQjUlbKj}%kn83!&raLIWCYR6~`iodb@f&1rY;aIdK# zHh;z2bOBT076R2*Hlycuu4TXSv4)o291|O*3YyLhYAZ%1k_u4X94yQwU||F}zyluP zKObrzh6kUOMe=mzeq4T(2eBFpZ$~V2$nh)sPMk>RIY*PZW&7;#P+ zX={Tg^M=I03P|FEaRu2hD^S90T5C26Whu3}LyNgXMjqmzc+ z_550|db9chXncx668L06yy_ZY={0N76gEMP`gMZ+j2M0|{0`WFYN}8Y*07x+!i|~B zwaQ3fUoV-=ay04RtPQ6~b~lSz?iNS@j^Ff?Xht4{_9;EK;=l2oee-ZuM%sp({N6iC z-LfTmW4}+j6Ok=UJI~W;HS|6IhH6!18~E0*Ox|i*f3WMR7iMRux*VQV;C3~@@BB@f zKhEV06O;@c6-u-8*x6K-7`BUGwk3NBO1jwVzm(QQ;!(2dX5#m+D(jQ@{!`jU)G}TuES(qWGs5)tOyukaVeYnRXjkY8BkatY!m1OGq4=N zfiwhXuCmBs3_s!HScwGpARkD=kB;A!PVYvK>xQjRMI;ONUV5rJ%uZ<4P&ZppAQ#tNoj*tgzm*-Ku&&kpwL|WA0?G!;J6*Efr z(xQY-v#d_`*)K7pn@QXj>y8iKm=()=nDSo&6E~Pn;oo3a^bi9B2d%YnMAKu|QlR-jkn)Qew-6O?-e9;d-nb8ul7Q(+Wr=}SOP#8VjSo>v zrb_&pAEiz*9w-4!V~I5A-zfDkX&A}e({!_i#JtFz7}tN92;ELeBTkt5A9Bdnf~RV0 zS*Jqfrc0?a{qQh@jw~(pSua&98||%GnowQ{lTK>kYCQ0`-wKu~On07Qh zT%v)+NbkCZWatKqUvq#8u%)A{Td2|wP5sE&nhC1*kF}+#z2SDXy$IP^T59pNFin8w zL!dw|r+mZI3FnIR#uo#)THOGbV(oDNwLN57OI?2r<`vEl>NA&j)}Hu@=}EvlCnNnS zSgoLQz8p%!SkWu-Tep!LIy^d{cZRZOCM@OoZPDk$s=?7V_Wg6wSm|J6TsUSbg}P0y z;C!kTMhr8ad1oJH*F;aS+gkHRbo_qESqVc-RB9mxs<^!A5-x;Vl%+6OOyrxV;^hg(#T{vzpCP5{Q6W zq9#_vK%x`|MM4S)5F8E=Ow1D!EO-VEY_!m$BSu=8;P<7bj4Dm2$fk2h_?9Jn(^z>U zU!+`hx(e?UT5-H}6+-Vtr?#uJM6G21Ctrn8l1c=j`mObk)0)?5U8iLY1QAlzu{P18 zO4hL@4OvxQ4Mp8%N*X?~u_$WfU=@9-*;tU6x5{Tx@qRFA^ai8*hVgH@s=Xa|icI27 zaOwa(Bl_+??NGWLvxZ4Npa3^i!XSt&Ip8&aE!9o$OyuT#z+cCa1(3cz*Lof{Mk9wU zg9g^@N5JY(#dj8YWkjuf$?H2#97up0>+oV438w$84+lBeH2Iae1M6{?S&b;J=Kb%M zGI!U^czU-lDF3@{5@UKuZcMV&o+`>iu!#-BaM=gVMvaOb>X}*NCf?6}<*&o2QA1y< z-KwRC7*@R4PLzcF#ws>%W4LE_Rklfu>8rT&YRH~zCvu_7U`teCcr>OOyc;+5RvsjI zNNH7rGNw=uUcQwL6=x)+{jpK&U%hKA7uP?g6QC_$3tc{{UD6E4yO%%i9&i*~;9b$| zYOtScP=~Icapf^{OU!xAy)3u%Dwp7jFZL`vL={tcpkGnP-%+PxpcQ<`TkaIST`K1N2lJ5=Ac6hc3`DMqelb# z%l7W_r$)1d3y>pFM(|lAW$!X3J*UolEFBo=`zZ#Yc1D(md;%we?Z!&sy<&4nI;ePI z6@nJTSt&9K+7d4Cp2G|T5L-b(*z#d1?wu0h&;9Ta)Z#P*`UwODY1#?&fCU8^!rYHf za#F%|&V(+cyL2T8tAHERr#& zgUm3M23WJKvByZ8k`p1uO#f@^BrUaMZ%juL?(`~2USc9n&FcNEe=(3lIJ$PCN`(EAQ)nHkf5<9Ney zrYLScE*1g>%Jy1zaszRiI+q5DlnI%c#MS_^u?@sllsKDe>sWJRXn72u_AjK1C*J`^MWF~y5qO6tNg#w=5dNp-s1{1))v~Qh5e-So&rtv*z3d20jVh3 zga0EH)t+Jd|CNeTPr3MaDyqHK_P?Z}%*_9tit2^LNz#RpW-Sy&m1ZsGFBEM$NrNHF z>S$~JK$!;I(c{RCdrf)0L310f(k{dnFMlieG*Yx;%!T*!L8i8wad%xV(u-`^8~jVn4$Ibg0^ke?r(o+FDSws#v`r)mPgN4$bFxT;s3wMl0IG z9<;hvBLEU!fImswmRE<}GBU`ieYTFfjtk_sD>qG2pwKo&4;@+zjBSjQFKF8va3{g5QK+*iai_wn$0rQJ0A^Y@h_E{zsy!$@ovAO5SLh>O@F_T%8JJ zr@A8K*y7Z<{3?@ZhLEQf*TUmK)tS!~bQsBuGRfXJMaa*ozpGL__4+sXu3mpvCOyz4 z$Tm9hYk)h^Jr+>iH)NpgH-2^)4~;#0kfCfqI>`f&PPzu!+tL{70?1Zpr(*i}D+`aD ztydt%ut+*%F2D`o5p+YzHK`9X(ELn^txwM?o}m(%l>DkRy$up`-*Q zL4;A50fbpH?1q;F1Y;HbS92NtRkU@w@J{2NYfU6?d zTPePr=n$9%bR!nv1NZ3pM1dC2dm|3yd;7WubC+tn4}Z)-2Xzd{8@PDDrGoaUrun(4Z(YF2Isl3H3

%S!DMq;%7NN9Mg2lnF|D%(}gAFnS~JRqsvKl9Qme z8nJ$otzF@eB;B65`{!mw|pbOxY6qF8S z5;t;JTac^R**oMY|8edo;REA(&I2+F@lkc4Br7Eou!uN83{pyQ;FiGhPS*y?@`z^vUBBpc@Q>cP)KMW~sK_Qm^)b@-@UKzWY}jAwl~*V6LY{qj;R)QJv2j>PZf8%=uJ?;mUFc^6hz{`X9;^sdPfn_M+eS+%w)hxln47aN!{(L(EJ*s!LZI;eLsmkVZjg`a@Z{#X6Lk;SvgZ95=7W8J}$I}*K{~vz+WypH}!%4Y$ zX>gbSpDK&R*6r^DnOpjMCmX)3= z-k6$Z%QnhRbhv$wry`76m%g}Ft`^c0;h~z#Jbt|Y&mu#nB>K#GAKMRw0-D?5;Bn{aRI8O9AeoXU%QAtW@PvaL_TbOIOpnslV zq|g43Xo-N=KcgjAiT{k2H2t?|N!(i(Wt2|{o?dg8uzOFj&elRLAYvqUccQSy#SA?> zkn?^DHGC_DF-YGd$&-XwJDA;+7FVs~5|mQcy`llIHx~aOIr)c`vdQu84UDRErL2G< zqO-TO=;xQ$a{JzU3e$CB|J?JxM;Qg3V9jk*uws3tq!9Dc{mD`tCif z6QBCQt259K)kPO8Sr;`jwL_#=5(fGLoj(UDOdHRliZ3S(Wo-UZqRcj45~hD%L@c{gTNqyI z*Mo@teDU>n^z_p!mD``6V8bT!QB7Uy%&fH<@|ygbNn=2W^|VLSU|c+ws+OvtDU=#> zQqNc*EUn|@5U&I*!f~I9{E8_kbd%Q}C;Vs%oBj+Tm%FoJI3hjDqX!o6a-w`nl2ZkRWbD6Wamlbw{^b$TxH5F(XBRqy|LD7>lW8kE zZmCg2e1{dO<}6hn3~T{oEij&rebJf3T!^U3=9TK0P~F@`4utt&LpjP+`G-q{?q4nu z8Zn*Vk-koGsU9)XygB8(_S;nl8Y}DJ3{el&=d!{FonEe)np3%!D>^>FIqYWiYDIWT z)qKsV(n{@v^{E<$Gz)*?HPcmA;D+?*L)n(}sg0ujy6qhWb^V6U03)BFeU=3yh+>mK zn-%+~Hr-cf*|H!HRJp^h!_29NQ7n5}#_wo)i!vWVv{($8V%ubzp?^upJjtd1t@~sC^(V2Id(@SEx|^UkThMg;j&; z!i`=r_aPHCU9<*cY1T>S$~nESOc+KuJ^h}RCtRyI4g-=}y`zOH+&&*GnW zl%n_Pop_XBHt>gEnW3e1ew)hHDG`e7MAAnB+33~S(6<^u;C;bA_JH((dP?XPSGXZc zU12=qG`@)vnsc4IZ)8ub)zJ0{xpqc$>0a=F#s}jai^4Bi@S;`C{zvDB_1AxOe!zUD z!3f)Ro(F8)DH~PDF{FlV+~qEn9mT>ZW^exhJQ=jnL<~$RAHH{I%tTl*jR_XCVeGj} z0Uo5S7!U*sWlexwE=Gw!mNnG81&NWn_^0B>h1IWsG=_@&v61i}E@GT%tdPG@(saSF zVcGChkVA~@hRc;P10YoHz00_$7M`u9tArw0AT44_(=AKKrTm#9Xd(Avl413)-7OGe0M1Os8w z+Q~4Zr%H{dNUYn#4?(8B$e9luFeTDu!y^4m17y60Sree#%P~M@L;skCS^7hDT*kj0 z0O8q3J_TyZfu1*dyxKVBi8~eB6dMnjse!YoAa>9 z1su*$(RHVDUf#KyHG(;^x1l2sKNdK^c2RBv;kVM!{z%87o_7@^fNc|5?V8>r&U&vrPUE=wa)w_Q@@J22%+=km9zH6X-= zV@(X%k6{O8{V6l-uKyZe5t|cbTNb+qRH2;=cmH`F@ZygJUicy4E%+|5^oiMQY5e{P z=xP}4jHcp>YnrAL1~tL&;MMJ~1{J5BFCu3Hi3a%4i+5`!jPPehQS+MO(vnupsLoo( zG!>4j^Ko6myy&auGYX{ zPfP#;THk!m%nsBfpeh&~?0r4w@#;ZtjS_*-GDHGB8#FZy^-C-i4fW4JZix|*R?^cg zzgr(1B#6u%w+~Hzz7{n1ty`7|bBssmh_-kPV}D1yeJ%$KI*pOH*_AGgTqLce*m`tfN}xd0Kg@> z1Q*l~=mr4p4ahzKaDhCDSg;r`ok{w;yN0@Hm{Ng{rvaTIJ%G*-FW|f1G{<^>Iz#UN zbcURI1giH8?x5fcO0K6q+`qmCgu5S;P z(T<$KIZ%mM&q)VWujvLvOOFhpsSAKg&29ix+T5T+f@wDVf!m(|xQ$Q{^a0sJWtNIi z_yf23Ker(GYwQ4;4va6%DpUBQH+ORo??NC;p=giuQ1gn(`hAU}YUH+To& zsMhv__0S z=cq&?IOr-kn55Tiq(&0xOyh5zL7{e(%^L!mwm%= z;`>nT5xS9h>Aw^j{B5@v1T(y7=1>ThO!2QQpib0XR7yr%V5PlWI&ZV_F@CBSQAZ^y zp`DOVaMJ8lkbYOI)N#yH=(^3+toe`zbTcJOoWs{QNz=T4BmtvTLw-dlV=pS&Qp`;R z{UoyZD0(|VeGKe4YiuiSH$|CBz2W;~8-#ufPFfY?45Gldlv2FW|JuvX9J&hVcD#wo9K0l|AT)xVuFogA&LG;4_M7a z7e?!ZfpJ{IWup5V#y%(q*4JyxdYb6yB6OX`4$9gh9n`M=w|%8TG2#^vb1jjkA^sK_ zRtBUmm94WekF>M+8EB*M(skK9=vw3OmP1VEkSqyxdq^mI*{<-|na64LbD|r%tvI>5 zrr(o0$kkOWMr4h7{JxbDnB(~WiVy#nJj5BVci96guj6c`}K0KPvRs2Vql@50cpaRWJ6fVvKuaGvEGjHy+#XETB0=CYzPAALv zvSny(*_ltn?it@oz;IF0;6Jc<`mN93&uztj1qY7drANPQ6_5YyfdL};_P^u{AC~?j zU+5z2PxB{V$jl;*cUiTVdFpZ)zgcDHmc+EJnctYW13={e-_bzQj%-fLb!EH7JGxwc z7xN)Tom053r8C9!gt$o8Y7%+IxlGFu6wZHp58Va+@+`ba$jJP%3@I+PwSfs_ysJn# z_w`2LDw^;G!JQ=k-G~fdC>O1yu1r3mt<~|GTRN`?*6?Ir{#vcHJ(hfA&nq4XRV`VeJ5ez$f;?MJ2^e7-jRsY=kBD3u9Mzhri&NjWQ*ltWP(CpkYZI-+q| z<>_3eyMWQ#ToKLx2+1W$R>UQ&s3t+1P1llTb|aITQLxF0F?>7{#KD)AFLYUBwNltr&Jo30(sQM zi83E|uIUkds5`nHS9P7NX%^)NMMxD}K*2GMoA?$*9F7w5$+P~(p(|AP`N$okbj^Tai zv=#Z^l3BOvjy?GO6>&K4iJUQ81&6k`e8QygVZAP;9V`)rC$a0QZmiV^mY+ zaEU+pjozF6AZx{H8rUm7c|o@W_tJjz)~$}WUkRv@oV4)~g`}Ve^orPzazW7}-;&Fm zb)76;QD9K4g#xy3B0!v$^*5EVBFLtNF8Qa2Zx>%R~8c(ag%s)$dRNCBxPF}2^ zRJI0&rQ*;j4y})SHV8#Zkr-&yUkY2o5@AXua;fO?_Q@A^8llW)0Ow#{@TcZ?mz2Ev z!Myqb`x}(#MNtQ0ma-B8r%xe`ahyYpSEm8oFABgv1sKSDNEM>*;@U|Pr-*z+-Xc>L zO_jr#ai|AsR_D0#0>9F_#0|PWizjEaz9u-w%MR}QRh{UcQam*8R;(OV$hRFJL+m2A zo+5X9pU1d(@9$Dw*J{>`=)Z=vWxV*T6Kp`@m$W|q4s zr)=TRnP0hC&9QGmx|hWZvW~K2w21G>D3a6~Bkhqc2a1Pq0T#uiYTT#$0qf3NW1w?v z9?j(wkz(GJK*74UB{$G)Z8x1js}h?O@8HFHW-T=GXxW>`N%XP6B+}E^VzemFbA!Vo zMNwob)Cnt1Sd?6?oT!Cx`@`kyx~!_%SQ-Qncyu+e%F_)_lOlKQ!vh-W5{<~2rxGUG z{-5bFm)=Up=FjlA`w6kjiH)*Jrt9I3l6z#sp}9tU;ZJ(qZBG^_GRYctVZVt;i6)k8 zM;zMny}EuhdY8-GLrZ@XAW%qN29IPqaNajw$b1sjm@WvxOv24ru%kO2b zezfqx!J{pol}9R=V$p$qn?X)2hz4+cQzN~d`O$S5I^zdjiaW&3Suz?~OwXm~kT4>^5j zs~*TqQQG{0IGXVi`B~=*zioZ%NmL1viAZuPrMFwnwnaxXh%fj@IdXF9E`b0T#&NC5)>dL$UV@Qc{rE(!!V ziJ)Hu6!&U)FUPN`21>Q(hlkLw@K$04QCECxzNfyQt0TXY^^G_nKI6|nb?Pvp(d;xP zvR%J8>oKUb+I*Xd7|EyQ0Ox)D0z)2{dTC(hp5PIru}-lz^Qbx6B&#coqTt+@cy>xf@kKezii2 zEalfS{K5($QO^p!l`rxWdiAw6UYI~hRtKnH8w1c5|Cg;OhjLvi`T<}=&t2>v(fl?A zH|!``G=3KhE0m$UEL;qqHXT#L@Vv1>IVT;6(<>;A25jk=Ti;tq}3+7F}4 z{;wv+NYr%?`us3weXs8W&-Lk*pZqwwE`=Z7I1>8r9G~AatsO9T+B{=hE|E1_=%i`0 z=-OIy3n(_HF;v~JqM3TT`+II{y`1>TgZW zZP;{3?uNTXy&=@~E!@o1x?@hl$I5=mAAM-r^1If-+*Uek@j0~p} z(<*M?o8qB=)>C$w-S&~xKr8NLx(hV9`u*IRq<>iw)o)-q1- zYK#);O~rixGX_!)jdyIS;j>K{j7NzXj#u8L9y$37IN4OS#JkA#n;FeKez4w^9Hwwl zA1sQOpJ}@cJ*ekg`TgBEyRY5Y-$#qyjTza))|we~?U-R>-8RB_dLr+}!O2qsc)-TN zrM1r8kaAbC2HV8`v`s?gfBEqwqA9#9=jgN`to1_@(oc`$*K2spt_q*(9CMF@71lEF!Q%EgZ!!Mk!vv0$dgMjqvQ z<>)O%D5$@+crE>-uMAAvduFEXFV&fKYA;~jO_naoDH|7y4m(*rx7(e&=9e)bj}|u) zNYA?nl5ZSyvk(=~Gd&&c zD%yE8(W5`-KYNgC;zYyIT%Yq!u!n0Lc}MT9vwvtahnN0tDgC1MTYnj`;+!c_vQ`UA zH^LbbLwW!zHSIV&F?DN)x}(Gb=M$s)QN|~vn9LWMdN0XFx4|^=(9ckx^uCW=CK^DG zaCBnGhq+7;n;?yxkpj_#gJi!^_K)B@2~4uSNG>$T8Bxcz6ZNXZdwdAT|+wWPTk*DJ+@5 zL~2Id{X;%_!+@qCipKg{JT-Q>cp90lKJWx(Yo;~dP@XH#6O8&BvP6P0bV#6)#|A}$h*nUr zrLK5HrPO^^6+k_hdMk)KMg7Re^kAn&?@O}s0e#~!m|Oi6J%B`<9>5Ro9)LS$*Etq#7zx93vch^;)?9<%@oBm*FV2B|Yxaw&N7p zC=Dx~Y0vtGaE~=lFu# z)x>ej>F}i%4VwEBWl=Xf0P5lPCm2>G4;KBnmG&nDX)ZW0W;g?so-L(*n84m+8AWL7 zI#fCZL`}z4Ts0jz^lKsVQV1bX@WGYZlMDlXi@ax~Q%~F*BK6PM1GqvxP@&#I^J)-Y zQmM`M`+Dr2?9#A%u_)2#;5(6{Z9oy240tkpnQOJ8*Sy;mdmPxxfNO=xvoSLpuUHT; zEJd&PDq#hCHmtaZuN$f27rW3!I&>a{x<%+{aN z9}v`#3NX0{LTOIabCP|a-Isk?)(3;6M+Yb8{`gd#0NgD52(YslnclJ31V5kg#`j}a?(RF1YePP*ahOS zKd6!UG<88&eaTl9^LQ;7)qXnF0orREsDEue+4w+%)pQZ#Hw5M1Z<=nJ@qqO*rWM|B;%Kns*lEE=RwEDEeo zENF{9wGdPG5Zph0nGrujU&bbSAb7y(lY+s4p?DKS`a}3EgKmQ@!-uWy8H)Xb{^R95 zrgqmJqyPdG13*!zcMO3+0t(c!FzFS}X>RX3aR9+I_;?aHZy{0-lH6~k{$H52!7ib# zcy83`1T_$Z@O8cEGoo>S?6x4$Azr~A17)h%xJD=l3ChH`D#bx^H5K7HDYBg&?QEHq z^{Gzx`Xugjthb%5oSn%v5js(ft-w}vb*n@4jb?&Rc;u(L$!ZW zgGOFEcDGb=srU!g-6z6l$zoK?X1q-9&(*BdCI^z`%srxdmqjm@1r3-V`Iqx2r+xIE zA-yi-0K#?vAzB}-$|p~q$^gE}>}WoNJ2!$YNHu$Ov;cv{00I0ZweW+VI%tO#ViJ@r zwhx9N?QUSrl+*kzua{)}(cfqljLkcB1+dtI-!Cv}mc!h>x#kYR_UbitdE=+;8Oo&D z6M!o)erfEOx?O_DX%RY=jM8V>b^JIJ(YxpruBO2hpy^Lw_O?{xFV=;f$}Ib8xI+|L zYu?MX)-QFAPWkTr^4(xZ|02BGCPU`7G_Bj_8k)^N^+{rf`9_|@W_sM;DyF!}`h|TO zJ8Ylmx$+$#iEW^TJont0)7RJFcSN1FDB(}hp}6${v4q)?s>gN23+m=HRq9Zm^&hQ) zn{EEt81FoZ{d9x!rO>0Taz>gN=imCgpfUbxi>Kt&A;!nRpnxkaz!u_pwn~}Dwu2q5 zGY(|isW_}=-F05#d%&|Qatfvqx4&^flzwC`LIG!y%CK!!-Ya*)dk0bB!h1Lv-B~{( z_HiQq;=#wijhv9;6QCih42A$n@prkd8sY!=C`qV2$5!+xBtBmEezOHN!&~h5XEQvv z0CAPmCdgafrn(AG+c~aM#$wO2GQ*g<8A`Q)rGIDcl11p|7!8mu^xdiVjOnwhDy&i9 zm1*;8v#QS^yy|@f^}@5xLT36N=Mm7+;wEx+L%kt`sXyL&jR4r8QBUy)Ol4XN=N;%C zslxole*YfX^!}UurWM^b)3ndt{GO)Ko5@mNi_%%yKQQ`S-ug3Z)ZmP?S`mCq8?awA zIpF+T%;~&{&3AOXjS|0;_8(p?;S3SmtBm-yqJoNM`uT*v8UR}yDMW*M-O(({$r~5^ z)d08?G}5S@i_1>?ZTEex2Evf*XO;#jjZBwZ^OVV%T>8m%}uX&brdPdeaq3eRcDjc z=I{KQ$;%RNI|jN{1K0Uk!;zFsjwN*dXQLb0(wl_ZH;s&{iGroMb4Kl~-2zTCDB^_O zqbU+Z%AU-kud-?J-5{r8>efZL94)+f&{wYLv)x}L4c`3C?sgB~F!A|{kslIaMV|5+d}xLk z9@Uw8JM6Ma)YmxzN?XdpestVp(q7J=d3yycJ4dA5oTsD{{Me12_kFCfs@HPJ$&244 z$2F9b-BVw=e@gP$$}h=1dk7K6W0_ z;ad_)vuC4fAVZy_L3Ez;uaBxvT4jbFn3291?cg*i5R7`k^TmPp6bM^j-l@hIO~%?u z@QwZhtv+Tb*iNIILabyz0AOg?_zF!1-`H-sKl~)qf~ys_W?sqy`JrG*uI*9V3;n=w zA%5%Y4>jf4HqwhK6^K1`XEsl*9aK5Dx>SYuXBe;hg7&`--O~7A&+N$5LqQ?R=LH+= zOXP@Cv&D1|K76M|e6||X;QaK+E9Df&YLI$K!rX#8)CA3E-n?fIRygX*s zOvQ#x0q+{f4dC9|(|8SkL!NO66ND@vPY5 z*pFbl_zI0?0Jc$^bkc5;NHVEryl7^%BHoM_J`2Q3r=m(~kY7&*r63=a ze$NH!zJ=DUk6-l1r(j|RUDl5zSF_bP16Io-q75by0?+)yvR#v3S+%*TD~ZqAI2AW3 zYD%+5tOc7xLLsZvaE+uY z6*+VWmo|ul2&8|+L|i7|BW?w{LVa!Wh5FjLfm$omYYda!&4iF67<|D$;6XFQLqV;4 zmvET?Z>_s$#8}nNLK+tNb5wJZY=!A1)&2rg18(M9N}_`}|46j=4hd^Y#(;Nj=~r-a zNr>O%gd#|Hd^kT;y|>cBv`Gr~%#@2g>2x>d_Ra}MkVBHS9u=E#bq;1@U*FLsj!d69 zfjY(1naP>;DvVot(<5t}Nd%H+1+qR3g!Ho=Fr~ z?Fv{jA5x>;#Mz!n9UF_q zp;|e5=gZVT(y%%d1NIa5nB219?yxpD1Pv%LBCcf8hXHY>!Y4FC&Xxz=pk$tqO`r`v zMX4|eN$7zMy1n^8^Z4ZOOi011@s$cd#U@&;p}$+rFr~rC1k1iBs?*e+`t}M*!KcmsP+3zHkD$6w=BGL= zKCX3PP@BnX*fc@qjySc?>R*;K0Se_FosSewFatpyzS0Ap&0$ z;DrEP(hjxeDe0x-{?ZM77<83!!I3Uw0`QmD7WUnpRduN;C`|{c&S5Ikul2>m#2T6G zuGHHo=mXDCSG=Qk&j>tloB4avY30f#Pf*maLpT263Hd8Qw>y}kBzO59N44+mKv5*A zr?z5Z?t>t`{SBUrp!eZw;5DKCh2spy$H!+Us6&*Fa9+Gux+AwEG*V`ab;uLf1kOD6hX61g(NoiAc&K4hCOIx0 zWrduC6y{ZlAzt;t_{kN_ptwuKK@NxZ#bamlk*X19Duhcpj7tu_t_iN$Ii$r#DXdY< zB8zT7H9;g%3`e&S#XbXL=PVTdN;E;_wf0;Ds=?}t@6C_CIfz7SZNd=xi=DO;Sr=&-~$cuRv)t>NYY$tp3!UyZd%Zl^=$1EK zuZPL+z#_G>!;5yc$bU5ix>LZmuXl}(3Ftpo&7X;(d0ybI@8A7!_uwyxjbmkcaSe(3 zGpwtW1>>h5pPICW!{QXBd%KAI==;VH_d#UTpd}5+$)vR0+NJseEA#SLwcTLWNtcOa zlopwogDnlIlkCYQpyo5Cn|GfaWxcY8BZsxbnrqh|o1CFHB;dI`=AzvKberW6@21ZP zcD)^2DP9geOtB9dWqDF@B&3TruxG#AceaTZODJ$0d6k!skfFVBwSg(`8Oxwpoh>G~mD@5a-7{nB^VytjLB`lrkvby|ArUq^l>11la>0R#ASOzn@Nf<)M=HBnm&19#Xm_=n1)O7Q$BJZ5<3&m}gjC#T4sph%WOe4NVndk{y{6c9b zarzk@qvLrO(%3-|e5xO>DU6VlUcih+=0;Oa*>SwN^&_(ty)b&9_Oj z^~{ixzVe#z=NXhv6_a)nud3-$oKC+FVdGm_E#q4iHDi~bGaTY6l7g(_n?VVAo}L_% z@;L`cvtKs87dv{TrT#whJjW}s&g^=dSivY^>R7M3so4MR+h3yY{s&VuXh=qvrLZey z6?=COm7lf?XI<4J+j!!}B3t5yFa2PKPdxOU5*;Yl*M(&|-%DKclxfNc=S-{jN z{bs~@DKiG>)6~tA;UsWvZ#^uKaHuKdMzI-{#O(bE9GcxsA2fJ|KWQOV*Vygn3dOAa zSw^04JItB_s`njXB@Ng>zQ(nFMVt4*9#tFIHo7=3(F}Dp>d%b z5z5PIR~e2l&SoX4j4r>20o}w{l(xn>nErbYQgS-Ob`hRV+*%Es;v_<;F@nZMlHJ2` zL>{rmtXj2~p#V|FMk9)IQsqi*VW7jUW>n8Z)N@!&9J`PZBg5%!rU$y3l+Yf$d7@3g z7xh@t%e;u>5g)W^ zetV9Y5I^sS3m^|0Q3D~7TMbz22noTiMW84XzQ0n1>nHo*q7Oyt+;*zFpBTK|8igq> zm?Y4cX+2I@7E#*^YVs0%>+QF9xqMy?)8;O#3-C=GM8wf1LcY2OEjsJ)O~WLNG&(~0 z4B)!;`GGw>-ZdH)e{d2UZtTR?TN^t;&88=kGzDF!th1<9ydI^uk25=PORE5LH1}mB z6wQ3UG+Jhs3!JzFP!cGRVuTzTD?#nI)164v&b(KjdNc1doS)Bz&2@>Au}sE{rE5r1 zoTRsEI*C`#RH9Yr(sdy7TqhML<-eLu#6^atZn;db@y4sQY8p8~IiVH0w9Lx}vE7&N z)VFbDRCN4_F{$O;7(c0%=Hh-F=sek*5sAZq$WrI zo2=TQxT88^BKA{&R&ftpT#8LZ=^3S|uDeb-M61Pyf88nTc)ZsLCO0!h7NoRSbCVI5 zr9*o}VL@klk!*{CgI&{!K}#Pih(dSRdqrZ8b8VD#t((!QF#uM`J-8{l3*C| z#tOU4wiB-JMfyu>mckagYL>u>M3Eef5jFY-w`em0O7AR~Jol=QqXCpAn)eCn3D%y; zpO;>#W87yaerg;~%*fn<`T2r2Wx0A zrTV65=}8)T4d|XTdlcuA66G^-p=aDlUHnZzYu$2ce}!!rBR)QQlzc|KzTsVU9-?QK zBv>{^xVf`L&<}j>ffA$&z)MhkjX|Ie4t|(NjC5ED`>~eo$jKdI9=tml?hP*=dOv;a z)~0*1xA(X=&chSZZxp}g!$t0>k$Nh~*PsO9CNDU3Xc)_FJfI~&k*XXzGA!F%0xI(( ztiI#EB(FEMn|&W^fO)9W_{0yte$Ri)f5b&UR{ygnEWsFNE`Is50njbwhwn`I z1G{)+OV|6y6F1&B5+fuEeAErMQv2`jvs&z7H$~$Zl^3_L=-G%{Pc#kQ$KdcEm_A(Q^+*`(8sV>E0smFz z?|KqrT785$KiT+c4%0XnK^-+;q6qB-L5^Yp3vC{3-``?cVRiF*3LfU3j#2P`40ns2 z#y^9bz~T7&5_}13TBt{m`iZsw;j|%OpVK_KKih~7T=S`MAcEnRa|gFOd;22$=d**< z2ujRGufe7S{_n68BO+N%(>i6%mpC^@B7Z6=MkN>obOET}ONJ*W(_pYq(U}cHYyHDF zP@xbtNS|eW9Dm?$$sl-yL4J=O%vt_GK?FRqF>8rmyGp|C_r;KPEc<(aWxm7v%A5Xb zj79SSPi+Zvn3i+GL}BSbN$@Qvsllkn7>UQ4$oq+48t8J@eFw&(T;do7(c2d&P!|-r`_< z4^-&W-Gd7KKS$D4|0wiB{AIeQrY}&Rn;9P88;9R^q$qZ8L*i8QnJ%q5_nqPJTG>$lo zq19tlU@5W*;fWzyQ)N!FyF8y>>h`5G$}%aolVV{G)jNUc zBs8!}0D*kzh5hG}w&6s^kCUo2&L&PRd)0v|G7oDUR*2hvdLD!)P{oN0}F9~}F2KOi1+pcNK;T8`i;l`QRgoh5l zh1$-?CT}{!IrzFGB-;As@_q_$ALcn6l5C60QGd5(l0X538O(qfSOh_|d|2MD+DOU)LQC3Skw9U2Y= zb$VBt1SdT6k*^>z7M=?}DtdAE@cf+kmb7)Hd9bm1OX7E?VQY0HTr6?(K z@Q$r{B03y20}G&{t_ieiSBfIYPBVb~g?1oD4*qHvwA@S?^0?OaIgaw0%qzb3>7h7%-+Qs5-1kNhGEn29){ z96IzcGR2R{a?x{wY5R`zpRB^=>JIiERD^xERR&Mpc`K4U}{Ux>B_`jjdRbp zHQO#7Mnv^I&=FxANNhOEEfH19{;qY-FNeB~)6Q5<@BgjV;>iIYIgmr{Q)@nh$GqxU zIdG)$*;e6ACHdw=MHuZVcs6oCNDeyE zMH&N&c>hWGQqbE&1B$pCAD3GCo-8%NwB_A0eTt9=_G4@tg*Wm%yat%j0YZ65x~H{+ zlEATVkMmY7o*b1XmpM6jjJUtVt#^PNbxvOno-B4gXil;Dm`bygavbaK+ z{6nZINdO#v&t+Sg{^muEsW%sYTCmNHwW|pt8|^r!C#1XlZ`wZDORU;hEzpkw%Qaik zFB{6sCV-!pCPqP>v9e|t3N|eEeX>V2Tm5S(PUNbxm{fV`9IPU2XluzkBN;Nf8&A)Z8jWH}g~fI-jEZtmRag>v3QmK0K%U zWR=78s^(R`_w8DAKi z8A$kCu|CX3rsvHjDyI;B#7UYxC?boM)2r{f-&T~;`YD4^_El;LAGU~PXpQiN%D zKVn*8|2UFOTUFh9WiDR51E!}~eM+o%uVU+|Ml?P)sA3N@iat+hnjua52ZJ56x}C*B zO{pjkGa3!cgw7sqbU)D!14KlemkFY2fC{YN>0bcwPVijF`?Idz%PY#Z#eo}{o~(y) z&Hi(d;nzRp7^z3mtM>Er`M8Rz&UM=G(oACbI4MhS$TR^=(geq#6Co}LJ%a&lN+pJ6=>*_pnPeSS+~>N4 zx*zYlgk~x)774kY*;?r2oV<3)0kVaDP@}I{y%(ris6yry_p!7|vML$@(+PA~kL5v)nWy<3Tdwk0PdnWXlKAyL|=YFRfpenX2@XEqB+Cfp7D5wJ3tvY*jEINxUud1=H&`l@Cu7E2tzW= zX_78l@gk%v)x_mEi+*jSOKj_OI5%98`CF#Oh7p zH7qH=bu{}95LwxQv;x+jDmuovnDM!mnUfezns(tspCKWi5kx%2bCKSFBIWMPF6!Jt z21HAY4tP1j3Re?aK$M_}+wVCd3UQmuEI*3A!w(C4+?V2GmUd%z!)IdeFFGsbTB&eC zUU<^_>THEae^$9S<)BeOSxcXTcvf0W@zBWy&xk!JLjGJzgrz|#nI}M|KnaTB2f{~_ zrP%t22;~i)w8WgQ;B|ip@s`)jUFdTiMC{VbV8~`Cz2hJkV$~y>xGjNq+Vyg_?XADnC5fF~Qj}^UB~xetF8{O^8`Aia4WMUzg6f{7_MtL#>yX zL*?j8k1mb#I|hP0@AMt(3{&aNQHo7|L+QR!)&uBHe8|6PY$_fnxPy6qDa&GiHQwvj zNBNx4R5ukNc>PjH$!Nkg$j5`vp!J1J&xIw38(v%BJt~~!ja0&KOE`MOqPOo0AyVsn zL=yLdEQ=4c#NdoM3&bYdF`0n4Vi8$I?VR(D+%PNn8=VfI9^6hx7^xRP+eDKAh|B7fx6eYvZ#wnnWlr8&=sfCi*V0{!)y1`^=ulk2e zZ9Yz%U?mZM-0OWCe2c^)GC$-DHCR;AfOmk=iP*ZG86TP2#BTYXEM^x2&>%rbAXM`c z4&P%KIwAb-5-b**lsV01M^Ngues3(5TW>G_qO+|+`lJL~G_jmALzZQ(&;C1|nCHvL z>Jc$KpzyX@n(}b% zTqKEQ*UEBmu5Aq)N4bHDXXRf?&YyoOA1yViYN2SQsHS_et(((F^$buL7&2LSN^BIF7Z|AXEZzUYT4~s9f$sjlGsx)Jdx1uwxQt?=v?D? zzS3^8Kc|ic;|f+CW#4Sx@M{x)X5{Bz3KEj_G)?$fn*MlntBx3n%?~5qmd_ca^k(2(n1cc z;FcEag}S3u*&N&93a=1|*}!B72D{vkUr4=3=Bq^wcjQQ7=VUN zWLUnj@%H~`DgMdI*Xjhh7nHcw9$+jZ5(*8tT3V_Lw9sZ4i=DSnd}lg1ySeu~(P7l( zUFndM(XN!LY_tEx|L}A?{;Hu=GGYR%%ozs$odhqM6Y0ym4uS56*+^k`aDC+-zej4? z!@MnO;6u>Vg@ou3`+_XFP{I%La|hspBzQfbRA>cAf*%h4UnO{b07!!GnCy#u1Hkqi z)l5IA=!nA8^kN5uN5a9bldA_V^nGU8Ek3@NCXtDwA0hd^ME%o3BpN61RF6Lo1v1n& zAb8mZpIC59E`f+Jh-vE~%!!@Bt7!n`2t*9CKR3Eb;~Ry5wGXd~tNb zgquip)@<5yGqc($Q&nNh+`dwGj1;MHtrlPzPDK!rUcS^>djcT~eu4A~2ab5fZS5#8SG=mKk<+wwS}qE_)>*EqYe=w4X$pJh8FM(x|U*|q>d-t zuMV#pniI_K$ad?Fr+xFwd=bOpm^erAxx>@gx=c?ccFBd+bi#6FSP^_D8Bd>c0aL;6 z8(+hq_eA)m^w>&R)#3B;$tRt*Q9zuBtTaY(G?0vF5&k8>*uhMw^?UkSv&}V|6!6>;!7bB@v&- z1+3AjCmdAUIt5BPM^I?)1o0v4aKWD2Cx^@Ky5j&a7Kf*I<&`vyTvSWdBX-gW=cTX) z;!4z`?~OLf-`XzSnyX)0eLjw)f-R01&NWA{f5Y?rn%+;x95&&g%hd#*fG^Zkt%VS{ zrfJXF+|%RYs7V1vPqBssLHm6nhM z(07Fe#^t#p|K?BCEEU&%1e(U{A9WUr)xbGiQmPf$5bwe0I~)88N{39~j~Nf_T$#}5 z(jE^y!h&Aq-=TU3BQ>GH8+8iapVpD46a|BWVw=oZFgG0@MSQ1IZtXUlA=}$tMCaO$d`9dFSv` zfj!do(;%T~kJHg09Z>d+8BqU)?wPS|5VzRf(PM6?AxFdtfX1c6DIhwsj``(jKcL%uOo9L#UB5f%R(F|)CV~il} z=&67gll6%Kh=L$VU;=U>KxV6L-{H7TbtaJX$||`%57YT3D(*`BvbzLw|OImWyw~<(m{+`jM#e4!v?N05$-S%V34mB!PfL z5_pEvJ{(wi*5|;N8{2BQ7Q+{pctN$Rjgf-OtA*bgbB7uI7k8~#ClH5Sig zyaT%RyB@6IutPkDP>RY&DCaI3E;E$v6gT|pC#6xeXQ+4qI;OCWpUd^@>8sV>Tr_n# zT|n5B|81&ryvUfEt@~#>=|8Y3UxFLm-`F(MG$ZRr3Hn_jH>|r3x%`%>S`f26Jb{=6J7Gv*LC z6W7(auKDHe;;AXBv75V6jjpc4g6cYdMCQ-YsD68WbIEM;(4tS34CX#E?Jze8fA#jX zNq&oO9X&f4QXw3##R-ATo)_D)ZMy`ge7xOe@Blofw)H@zd`_NdnMl!2_TX9Z_pm5( z^OP4H;|C!t{2JcBos4ah-}2Jjs(7;4wB6Ru(~U+({}WWH`t{e-pA@cPel*fs-*>wW zox4^2tG@BQ*_`O&f7drqSpU*D;&2*Cum_w!J(?AR^bNge9^}5yb0(M7{Cz(7i1{ci z0JLa0d)|4j5}9o*#GO}aHv4>VtN%W#>Q}1{{fCoLT;u(Kb(vS1Ad(mg8~r~ zkr?tpkxRrJ#kszpAh~-8fjA*&-4c+Ae|s6jL0-ltKcQy^KB(B1t$=#yldLN5gH2=_ z=V?rMJ}<$>Q@|GSw|54klzC)g(VoYXlV#?Hs$^41s|M2PICsm{{cC@x}H zBv)Ip;nXah9FL)JO_#kl444zq3DEZbfI)wN&{1L?C~#+9en17VAha&S66*y;`m%s? zTgm*H+~2(*9}yc3Fzf^LJbtzDCBWMv4ngaQEtmJk@yynE3swivxW^(YL;8SciVc8M z4eiLE`iF&zzT)CAkUi0o|8NqX{PAD@MuQ>|=}TyX9nwBwt`XIMAD1Gsqh;xxO2bOz zGEQN09MWXz(PZMp?dtN>0|AC#K+^}XVTX}D=Jo)9t@)rfI7D(J%#1Nj%+~8|B{o#}GpHX)| zrT@mah@vPu0D7E!qfeSgaa1#*BFUG?vwnaaFsk(+d6m>h6HsYeC6TY%-Y!WaM*a9g zhYW`TnQ|d`w2bmA2#*2ue8J+-^7-re!tmGg<%a05W05TEMohzev_=9NS0+x*EopSPbr_jTu`z6RPtntxp%E#>iD)`ehFhyCJP|OGEn(KP z0hL~($pl%fjbV`zp7O0T)9P4EnTbNv;tQ`4heyG!DTJvS?c9a1gUQw8Q! zmdzj?oNh#f&7dfLZ6LrBtN2)*IQYsJdwmCCqMf_r$oMO|_dWE^0|4;B$M^>Niu`Ls{hVl?vV3OLJlwb5s`u+ zC8?i;=3?21ULB%W782E=ad_g4h-689>=qcBw8e@h+3n(S(1t%r-OEig@`pjhTqAa- zmVYO9<%|#Ldd>KlTC!)t;!Rp}Gl6i6!$x%g9B=S}al@Ru;vxQWuB#jvci0E=<0V^5 zzmF^#5|R%dDx=pE974ZGI>?m`@rYap=zEDJ>qfRGxsQi@G*u9kdRaKJE?~J7#3*7- zq1O)k5XxX!%`>2^9@%bG@_~?(A9mVA_1Y1&p}7@zFUu{a1`&JI>|$5=+V>Pn7)CG} zyfSr6mZ)U^VQ;x>Xdfhc@>Qb6p7fahf**}_HgZ~CvcEf(iZdJIvSnDuRxT#wy~ao}Hl!EG zq^GUEvY|z&Bt;#AiNI-ND*|IH;DpN+FHLnttwKsG<^ppb9fHV();(W6C%2qGoAfV) zT4yqih1RIK^Aq#JID8{K{uQUbT85(&<0tF2l+H7$&DDRJJAQNE5FBZ*Wo12lfHE6b zQA+rc75`=KaM`cXU0G8;S~IrpS}cmVhQ(z6b#+-=YvC9NhwGA5qbX2)Z4in}pSf1V z2w|^G1}S@a)USPsbj9Kt_-ThIvdqr;)hImWpW%$@6>s2Q(TpLFNVIX9eiM`8NB9u4 z)mNZP)pGrs+ftOA$FdVidw-!XvTx1vmt<2-s`cu}8dJ?*AC0^vS=o*MQ!c}ecb(sW zUmWrq3BYDb>q+}2X)TUw(dB0CjR;j3!3h-1*a)_d;D437w);PlL}CF9cmpenW*gri z=hNQ}`hJ1)35K{$IeqtroIlfWdrh&{4xWqNzx5gxI6<(rc#hp7f%mKqsnYpeGmU~y ze_Hr|>)9ZnB-=7{a<0o!9BY&gQR+h`6KH zutC(($G@YFb^(#TU^}Lu{0SyXXH)%|j==l(c%4E797-R!sG9!8r95yXB@b|fGGHSh z?%>lmFg*fF8*R=W#7k%gEL7M`{7AvN4-!12D|3T}sJ((%lu_smTs@HS2kg6lNLPsn zBZg8C6Bs%+DpH$NH3a7mY@=B@%-eqra5()>0gk#np8qw#VUtAi{&#>Q+0|q<{2u|1 zXz#+}hlIb$qr^d)HxPNW3nGsa=R*ET9`O_^@0q>$hWf=YXKenY9mQbJ59(3XU_7$)P;=Xor(9Qao33^m>)!k{y)4J=cM5u3p#da*l9*$C19>dsW?#^42)Ne2b*r5 z?+amk<-7VxY%%vQ=%HQA~N6eOV4iae&;*AUUj#Hq;ru(-lYhDU96(?+qE?wmrw_qrGoYOiq)Wz~O}$;I`qt%E`d>dshGU zjUCb9OpU;TcdKAv@#W;^q4Bwrr&Nh$LU_N`J|Gx=s4m(DR(bX? zb+-vkp-otSD{p4(A?G$rjVT=b5A9TPwHX)(B=wXCTR>4fQ!IETX!2Jxju6pP;yP7s z`}0momZ|7Q%!O!R<7Rl99Th>U4kb*4FDb2j@@ev3MEwdvaM(`;J|>2gUH3?y4a3ks zjr%|aK?7=plyN7Ca}1dV6rTv}1nrS1yKdTZNqK)FsIlct2{MBvw)p05qIah1HCBE_ ziAK9ZDz%27wM~K)Pv|eMzt>VmybtC=fSm`>92X4UglP|0oX3N64c2>|uE7EkOZ)MK z6xV9M~z#}M8kVKBfbOyb_On!)WI?@POL5~V< zDag}*Mc~Bi3C?zfI-Q$DJ-m+&SP&58uhb*P$VtCyYhrbamX8To=MjX^rDNP%X^pV% zlMP)F^EF?*UtqL*(&Ku{Sc3MsmDM~y((s+M=qiW(+>9^R$!Pp6F!ie4Z zf&D^wHm!tjXAbhMU)RP|ilp6~pI;iYf_5~#D1OQeM#_mVxI2P#bXZwEYpeW`n0NoY zd|qs7kjK+}sYe!KrLW#$TIrE1A?bJbO4gBU&sNb-QuC!tADx^?ve@hCfDh-bm&u-X zOW3oEu2^lN#_pFs7`vBtCc{_%=Bp21tdEQ5s(W{RpDU^;f4)S0$msrKdLhQ?}cv zqG2`*l$ZS1d|u6`O{EJ0YsI6?6oAgOhPm|2iluK-PMy3Rj816H8LISc!bxRxxqlEt zkz3i42k6UX$(QYo9X{ApwB!zsic$Te+}TO-TLWmLsl}k#A;cjExYcZmPAy9tq#()7 zer2Jw$dub1n9yaRH|cl}Y-~g>X51K8aatY0Tah6Rc33p>R2APu#;9e;+IDq^ETeyE zVA67*#G}V*H@MlE(tR28%_B$xDzbE+N!iN+mD}&qG(q+xec4CZ~ znVFfHDQ3sa%*@Qp%#1O|j4`tvGc(gu&Y3fF=H9vW?yL7hDwUwLrM*<@?ytL7%FiS7 zjuY%9$cJsy-|-3WNZ=qj@z2Z#p*(2e!wzpqA9aZ-XUG`Aue|(Uzjv|5SG?565ZjPW zOx=f!naU<9mR|x(`eQE>zkhyDB%Qc^JEzMgQL-?w09lCW$9>%XR&VTw^wU(F zTAc`m2W7j~a~`vAiS5AT*!Q#~iG^Gn#{gAs87xQh+lA@Em_A!MbFoz`zZnhQ@6ZZn4iaNdxx$D>kzt#vD3jbg@q^My8gm^f_-351oh9_kh7 zg<|Hw`DR4Wv9qQrgTlBgD*G!LN7TC@(UlpHKAX@!S{y|*@+dQ-qAHf8T&$Ga$1*Az zyfci6BD%aYgo}abM;b?j_%wCNz5NW)|0=%_AfREO6W~{mO#B!#c%z*~FOFXbwe&_x zpqNxFHk}*7U^jChV?2rE=UsP5S;miSGfv^s?A10P=)q8Ov@lRVHonhwrIlYtVC;Ew zXkXQy$k}2YS%p_}f-RNhwZG@#+~Ixy1QWvcyn7SD{&bo@tcGmP0$nz~T(DQ90FVA=KHkp!=6!QsmpZgSIh|V0W z%Xv9(@4M%f%hK0M+{e$EXUA?lU)Z_@nROsdD*SwkDuBIoZA9gB76FpDSI~)d_TuUa zthSTGaqd$L6f^k4FT+U}3=388xfylTZrKho3S7xk0^Cdzi}^=@_? zMRKq9BXb8c>cw^@u=JzYEDCelIWm4t{AOt#nOM(FHuM)#>HSby4Aai@Qzj5Mc%n2`p0ne)*3(0QS)ccNO@cPkr;e z(&~STkYHiR7$jhT%;$gbkGaP&<=vsIWJ zW@e}m%AU}~Suj&JJt1FW5Cu}oFf|wR4-vNe{%@Rx;~~fCD*_<;C|^Lnh{Hw*oRRi?OeGTJ~m`SPXzE`2E^E*+;g`7)As-AEk`=Ne<;%1zvWmY$RKbuCuG-*bPPRzm+7aW zko;Y%t~H71a=CjvTumTkfJ*K0ygb~GZ#Q^VANvpDGRBXVZl;!Dy)pD>ro0|%^3S@vG$Jw)kh-RUFU zVlza^m$9R}Sg!@lmFZi8ULmw=?KjP~XPrC(QgsZ&Jg*luG^9~KreLE$AkwOy0EB0l zS79JyS_9wNQ!(B_0t^F0KLQg1^px!Yf;(>akP#@?)HUr>Qx%zVj<0MX&)&lH-E2d{ zH(HYzjJWBU1V+zDs7ziAYB$Nb!xIdS*7w~k0v7S9O=@*ZV!aL}^oZJ>5M>JBXHpnF z!1we+iSh@cgZ$|*e+s#ry{~B+r+X-*=2%Y!65Evu03;Ky&lze{wZ(j4j|KK%O z8eeT}guIy#yJN}F{Cv!OV<_>Xu>QzTm+={&)*fmVmvrWU`lG+q4Is__`udkNON{=r zE1y&dv)ZpJR9ued9L$%8V( zR|+GFcoL3*Bdj818B!3udXKxnk!`|$)*b;%J%m`i%iBPyX!5w1PHM+-3+icg#O7;hC8^JFdMhk|*14 zq|3dus*W%o2V2p-hwFs4(QpPQ}$Q^GzFJq;fYaacLv!x7ZPW5t}FdvYJQA3_nmFw;KJfg z!$H2tAU$BzVG{`QQ?%iiWrX5g4II5$_-BxqTAi+s!V57k3Aim)w`NBc&{9l(x6kRo z)JdLxaC-AUl)3hn38!0PJV(!8E@D3zaFxdffHSodl8L+VFf=K#3aLBT02K)8t<|Tr zc}$|`@ohZqC1Z?eLW&AVW#GwflrWC)-#eIe7{n>tegqyK2N;@i_9Hv3$(lRrm^hkp z8V<`dB|it%=U&9`|IBuc@J|O%0{nf3saY24a+|e?epY5XEO^|akS>wq7iNo0-H=QCigf@p;t+Pk5J|rTbUmvZUeKXFTvEiPILSl&fND_G@YXBYz1y4zgie zC$~7#u*3V33+y5kitTR$)8chrg&eN*yL?LT4!bIr0J~23d9pV-ygE5f-HpACiE%v3 zA@+LiC%FWOEUfP=%tt#-;TVSjFA!Gw!NIE9j#a#OMJMtDbJeainsqc39iTj9*Z5+5 z0}u{G#q1AsZ-vgt;o{6aVBtV7@c+&x;7yGP!vBp;)IXl@QCxk~rhdz4`&pBOH+-)+ ztEwuUq+G3$GtM$B8+9^-i!HWwGK7H(A54&n2S=H`?bde(AN-V82^mm7&<+7=OeB7Z z7j@9asUOQPj$HaEBUnf&6O$`|X!t9CDq}R00OZYJMn%z&a3fXe-15afEa=`qcD5?a zI3;bu+}H6NQuYI))+L=y&NHe=$SB6k^t7&X@7Ahg0Xb)0-=?#P(5iCsD z)xE_K8A>C@z57E);ru%jm;zC3#xa8#EPYru3*vlE4y->iU~OD1TsT;6EjINwQkX-K z|1va7uB`uxdF7>uhcz8F1UNvLu^S@N01l8He^BA80-Ps5NDk5nmm>^ZggsrXYkj32 z>ra6o01L>szbqgD=I%YU_#1)C^n=i`RG45-dMwrWAzMP9SYjtAL}|k0FB>S*C?%6= z2QQFOHlju47dfx6sC7w`R^bEkke{h#mlC0jo;6CR{c{U-q}=h`e><9!R!cCyn9aq> zq-C~e4+*#a^17JkKP?M?O*HVWM|_SOvXe1ML)cF5R9(bZIQ6bKk)W4H^XZHkiA^TR zzT}NGHu&ZMzWjxBanL9{V05_JpE5b~qg?IB@w}zwZlHD0$9TCKV3Pn2%QG9J@_AC}Mk&H+Z+~h%eKPEuFmBz#S%E~tV-6YWjQ}003{*v#c z3#WTjEl*U=BphxvzKb`xyV-bSMpW*3A-S8kdG{wHyMbe&%~@NA+;3Mc0|s4m&%7ix z-AY*Jw`&xa_UlE)C9}-GwN~LLb8V zkslLYvNkdFfL(P*lH^%~Vbl>RXVsUVT0FM&&lORJ6sBv;|} z+TAApuyqJ%zs3^v*_5F_kw*%$3!Zr#deqyfCmUxucKqs2HlBEEaalQWe)v1DAS+Qd zMO2TO0a6$ySZz>!U+~;>-VM8({KGM8kn5+7q@{fs)%iQT`sKfeSL-}~8)kJzn(9GG z_hp#*tBw#!xv_h0l)gPsA2hP@uSXjX`Ls2&u%D^R)`S4sbEkug5utJOV-KE}M(?g* z;2C=ROrZalmf5yeW?ZDgEARezZ7ov2?2${B9Lp*JBRl8g{%cV2#w-61_9jT za`4*(>`-w&5lbX@puNUY7a+l!HCALna|PfO2_w94w&~{} zYb!5osjcYvZ47r`;O@N!^gF$~etS7C!q)v@a}CZ{+dN;hh}*<5>7^f$FusOV)d+#& z6pmTwiFEG)8~6Yad=%gNc$CRfvgKIm z`-cPj1k1k9RGi?}4kxYvQ^${@8eXWnFYgp(h7$AVF9lr5%4@Ij7pgEk6`Xh-m9XJ` z`!c`$^6}Y8O3?yQIdw(k<*#7;MJ`a2bb6}ndCS)Lrn|_>YVUcXNV9b7+93SHZ!Y&?sEeE8L9@}ja2 z89a?bsf5mEU1nzV^1aU+`Fn-us-9s9J&CE&*(X(kitl{`G#lCzf!r2i#YrVJ>$yx* z`*MUF)0hP@OJdB5wWEaTKd}$j0MnaL|0iihBV209zB|xay#iqFLk+PW-ZLCO>kBCL zvzys=uIG58tesgRJ(tNC{>AXwtSqYH2)@}9(vOCfgHMa0Dl*Aq3Q9M5x8;Ee9ltC} zK=HgM+kZ44o3P>~NVm!=uMZwr#C*scnm4tv%iST=V-KhjiZ2NL{9?l)!6xTTGe%M+ z`2I)_MUUwvy}+zOtt4PiqvW`cO{$9KtAwvj^2xGMbHIPTq_fhOdYRc;w?`&$`eFNk zYIyr4lB$9kIbtpwP>>}%0UXUfKQv&|WU>PNZ8)TLbiRuG9#AvoKya?1l45`bEJXPP zQwFgdUhN6_2}G=%I*bvq$-JF8WlG!6hqYS`rNnI?d%7UeqmnUBA80u$MLwLR*(Q|d zf)Vi$V`_Bh9=Z0cfk-bWYiPFKOO-1Fv4HbnPMeul<;r8^fb*(v=e6}-=t#Qtby=Zq z%nftrt71t$HcUZX4^Gj*3=6Wf;)KQZ8X(>2s_O1IXj7(# zb5aI_B$|syFV;wbWxu-8lej${)BZa9aKz8p?=Hkp0Ns9U7NrcW4lm>8DV@AyBjP>s)bUZ1CnX~06vgJtT#H|}dAA9LJ2S3hs z(cMgudVOV;mZ%!QG&?{)d#wkZq~FAhW~Zum`VP`77d$r`!gp>p zQ^I~{k}i?0!)UQJrVf)2b?`P$I-K4P;Or_5(c;_3v-NOzK|(XoB-H}qxsF<-TMpE50K$|kH(h{d>+}Ukp#hn`D&NlA&r6<)NQM{@MXC5D{FYe;2?(=g- zCtKqJcB(j?s(*0-yR$#F<$=hc+rGoFUM+qnTd$Q@J4iDynp(B&95_nOQQ6ftdnKYa z{e+jmsrqe02ku2m))YI3>M`fVb6(ez_*BVkF2q2J=CxHvYr<*9hYnR0q0?_bMi#%u zqvfeoHBGs5d;a@YQs?GL=f=Nk`VNA1&jxeWtkH4%MH&cZlSuIM3#zxshU77c<851|0 zH5`5DFr_|u?=gB+-Vv$Y;MH7?e>!>}FXCw{Aa69KSFXT5^;t|`{i-YV(;iHvzsV3{SgDIg;ATn-F;@fKj$#pNc zB|9N2q(V7o8NkTL?yG9*l=8s6@28TOf%)0=Fc}Rmq#6EHy-?G~Ia7bOWEPo5c z)>4^R{U4obKK{l<2~;p-s>xhsie7>u#in;;iso|p%8s7fx;dcZ_+pTNz7Yd|^&EU} zv1OSY0?6&v+vxc&y}I3OklC!a4_chU+-g@$`G=L%*KW0C%DaEGW0=_&=igHYt+1!^FcF9yvb6Y-uYuHy!@T~ZDDUOVE7c6hQ9ENdi%_n3 znD4JV$e@0NPwm=1tpfAj6gVdGC+*Kdu<7_SU=jkm;K;;9eANjAar6V}1T%19CgS_V zEJBPq0Vd+3coVVWe5IK2X_3rA`Izx3y|K0s5{HJ>6eySR6!9@-Sc#Q1EK>Cn$wrG_ z@uv{767eS&*>OrgQ8|g`gLUGZB%D%{o-Zkz^i|Rsbj>h6EvM)5dX6e_{wTHMH2}&S z7SA|d#W}FCtJG{IR!}J?E*{dTC>&f8v3c{|TZc^Y8#e?hvL~S& zDlQ@<9Tqd{hHGXfnl{?N05k*q-1GLCUYLmYzubY)rBv@PIcW*$up_YSxq_7EYEPQb z)FGeHnw-8L60A#BUA0%Y^;)|-=p-e{PPU}Tj2zdY7PfYfstU?Pf)y=6Q8FMO8xoL@ zn>y7Yex~zDVUci(44`zUz+?MekXGh=*R_)LBB^$^sPxvPcEYx4?Y=Ql_syPJIt`Gc z(U-hia0Jb?FSiSq7Tie$q@GK@WnMS3?=iU0G0OvUfhpI+!$&DID~_P>UKF*7QvNV? zDpXubrb+{e=&&atlxv|_G^?d>A=MFhWTC|oya)CMdVfkp?YGazWsywjy%+{`GP1(_F4OZ8eaFg9dmGuRLBs0#l|{+eX( z8D*X|ZL@YF^WE?}C6X(pHe`?~iOP5-2s7=ls^G4+EHtI5oD4qWymO`Rv{ITRxek(& z?_Aq0guiQtA^#_1Kp&uyvECtb}!KPG=TAyVc!W^ZYP5Y$Xcv6&LK@hGgb=+S*zw(ilTYNRx% zImL%xHZ5W?k~3y8T8z~Zat^yC61w+v(f(Z=hJNFUHm=LAUaAl14C@DUhUo)3!-D_O z8K(ccGmP_D(|V1~3-%-W#f-5h7Eh7MG+QH~^ep>Zm*vDiyd zL+0h%ir6eK$`RT9oGoS)Y{fF(#5>xxD@qK!ZPH0$Cz!{zzZL?4SdJKV(B%gXCJ>L_=MLEWe;RvLScSnpIv+GJ7+Z zDT-nL8h)-Wnf~DeMMbaAFvte@K&ztcP~nk?LJMmDG=tLpWd^10{A~uUQV(MqL3>uU z<ztYlSEVA$L&}41s&se`LVj6=)b1IaQkTDu>uxvy~~I zhDrm?{md2(?KJ9Fvnt^}Y2Q)&G%RxS9$_pcLM5(s7f*%s{BIHL&*OhZu&w_Y!JdME z6?~%Au0N|F3tZ1Y8dZKOBMs!#(Y$b+?kRPik5Y1Z8!Wp$S)rtXsv4u5=uA6@{4A3?fN`}u{p4VX$}52JZpdL$}E|Ig90S!Ki@ z1?SO)K2t3u_X^#k#LWMS|HMTr7Z636I8u6fSWYaqD=WO`_WhV}byZ(lxBIi=oXzF8 zUv@CZyq)d!!LtS}b?X4k$dD0))RiybItnlf#4efUZIG4y;<&*RtOJ`bTeEtSoq3P* z;>mu>@q@VK$=dptEz2MAAZ+pflDdwj2BfYz_Z<_;ja$yDIo5G38FrPTH38vi=(-r}AEE0HJ`bObjlrnbh+ppyC2D?0cHYVM11cFsm{Z@Q_lS8> zO}Ueq9QFs>aB8uMk)1%TfT~CsyKCNf;csPcMHOL5w%kx64h8k8&@L&^fc%gzf}aTq zkKcV!pVbj*Y{S>X1QOA+fy8lYWtRQV!&@>Vu24rU9m73SbW7Z=AT1qsb@1W>Hn>NalAk%EIZ|mEbfn93=zm37;}I01@JGEq9s)o&eniqB^AbBqUfGYc2O` zbg~Zt^^AAeCgX|9NYSh28x{}>^b%eHM(`>;NfT$uW7DHZs)W<$SoL#Q6b;a>R-HkX zXw-wHuE#ZpygGa%vbxK~wMF%aBJcMze<-2s721oBGAmtV?3wvkjy}gFKeDCX8}3=S zysY2oNj7w^!P4M8E2QtOsQ@OE7G9J{`juaqJupfX&I_8J>bcy+jImE)yMJC!B?yN) zpGm>Z_L~ee3Kk#t9@i^{UD)Ij3h|(~P?+A_JkFkcNgK=>bBKXt<}rd)=5f7KWtURf zsqxThkz^6wxAcmz6+FvIcW$eZfGDY^Y_v`>XXIFpL-d)ADS__M78y0J!xmeAy2w)Q zPSfaJ6?$_o;)~jFV$L|VMdAl_@a$xydphy*_Z_I32Y?0?gk?l9~(g)`Ru$L|9(C}1ZU4I5D9F-9NK4ipV+bt^q5aI<~h zY?q+wE^8iii7NFH7-EYY}Ytpag&iTAX- zVtwm56NZn|h-DPi8Xos|xT;d#+xL?1m`uJHE|lRgD|$neUI{BzE&L`g9 zb6)&(AFZ-JMXJEU##w(VSfoU*KeHxWmjcrl_5?F(qu{ttMw+2-l5 zIW2yfZNiAi_NrBz@Vj6T_Pl7rW<9%a{v(t&g<=6DVz`}*u0OX%?si5VE=TnwzU1}B zx9im!(bVQ(FHvdW4+Cb`qAWf53y2%w=(tv*A;an)$g7|5cJHcm;Xi60Q3xSEUSVZUKud|WXqoNM?}B$7^7?ytly@Yjp5HXqZt`u=lK`zgccAb zTlt@yGR?38G^B>%Ok`TI$IaX zDe_ZH0|ws6tNh-u5!PB6Xv9wDAmG?fg;zTr_9r;68Tx*3vCRDS#2EN#MczQULqH=h zknyl}*v$_Wjh)N)ETOzY_AQ~_Q~2%yhk$Q3|Mxc=+9j!A372mwpL=ExX%jM2>ls$i zf@5oS_4db4&6iO=K_gQl^XNK_#~{Mrj+(B>(rY3uo@qzDnuNI-T!%h-uW4YgwH9z9 z7g}?tR`pi2^C82`D$WO3Qa@uzThLbp|I}OI9wF*&L=9c<2yRr!hM2(;_Q(jm_n_)WZbAnvqYZXR@E3K zv;@6k!BtOz4-HU`Q(y#c=mm|`G5Ix5DH&2I!L~`cnu|KPn0yXH6*EGeo#~S%{#>Jl z#D&n}o7$Y#o_R@9}SND z3~XPrTq=@4ml8Nxd_SG&FGfNOPWU-d6h;Ua?tWCZfp+amDbabWB4E)vyS`7IJ`SL< zW5}sfA=^2`k<=rYsH^Zex0+a2qu4FIg;pbiDnwhtJi^A^JnQ&lT#B%>XE%D0qP$L` ztRx@?{!35^(BFG;GbIaG-&p5K)@}H-Zgh~6YT<57l%=3$tJG-b`AIVC5{X}#s+G&4 z!w;HOtv(f(n|>+do3AAi1{TXvM!t3CD8P&I9pU#R3GePL#RC= z|5DtvGod7ut)Pdf?Z@*Y4z-qX-XS8O_|S_5Hbv3uY^wubuNqV71VgDUb&i#B3dH7= zM%HIRIIWsSkd7-v#ISbRk0I(i$jdtrWudtL%)&; zp6Nk^+Zhipo~FrHOZcWbP%^%v4^>O4(>w)d;c8U_ubI#7@-xR$+-r+sTL-crDCeQ+&to#LJGVV~mR!b1kU=lVQW@~z>W!)N=w>oD{8fE!IcQyr*Ot_26$rapA7 zKLaBVxzt+x_|M2NZaD+MLKpvO3+19D0LI#bh8ok}@tpWLF_QWOX4CKW{lf1(zfTMJx@DK5^?Ep2S~4ZKtE;MrQon zyjX9ZbKDOtVrJ~e-n%^;&lFPjWMP|~>-0m7FMP7QGEdjWjW{G>EV!b{mXD_?pZ8g} zk1hvy81L7sfg4#@qM@&N;kF7tNjH~(IQpXl#G^8-Sh`s!u7e?xF7}BjPHknwMb(8K z7`d_s%{|ElnubYV7k;uJ)Yp}q=0`3ccU5Pb+q zPx|Ov2l;V{Wh2Wqp}pUnh!U<$%W)ydJU8Os!z%_taX3wOR@hxfZ1$jHrutnkL5{jS z@W_jAMM-^+w5N#ovo{Uc)oFU*r;WoTjj>OiV~^d9=c(1VFS!}{C^lr0lpMi(A%FU= zaly9-jDi8)vz2yzb>wVt5q)$n{07jiQ;SoUWx1i&#M;R5bXWycbLMu7%ra z(>EPxpn(!P&LkHn*Y&#@9@y#R@i$Tp>MS zA->r-R1-)?Z0v&w{5T^suCs1$b^Cd!ns(f~%u<&FRm-vnZ;hmUhh>%>0#N}cTa~{d zoLN4#)>EvVkHM;v#Q&?fyrTf#+I^|DtlRIKz=X;+OUq9s!Z2VKmsOTY%P%4@2CBrWM^(?0(H1iCS#k6eEnXfKHUT6@0fSSu z7|w^EDD8c%ybb3E=cV`RvhQ0@>u3B*a3~94BO2NG#Z$*3RN2Tr0ZD*Mlc95_K^O>=6)?bu_On%ZnBfCu0jW67 z);&ij)nHp1b&tmL4On{>ejXcE;zlY9kHl8-wLmLpubkZDZni&rD#4 zUjalkuP(cJ^VI@$ye?L(vY;7%Jg`KX9>3yvISTP|gKs9Eacm)0k8hzCsD#Zi){<|b zA|q%#$~govX6&u2z)!(ble6V4=+Qos!sWVIMz>16+a=!_6iHA~c#H`!P*FowW8R`r zX$FA;I$(oEBC_^gv$U~(=e7u|+vA2HXaUPWz^Cw(fj6>((fGiG!GDAjxl^ixs7sE; z97wli3!TSb7>ZDzXx`wAQzi-WQ-qMjB>w~sT4bQgka)e7 z?O){m@Yv9_dO4CO(~+jZ8tEs~;@%)pYBoMG<}L`D<=DH{sV!q^t3-20`l~o-X=3Fl z;lA$v#ZxPNMRP5i0P!7`q57D9Y04XIkx~q@{zf`D*>_fCKGx3q*!2+xRVt%^Kl?(^ zZ^TjHix%y^V1@lHC{aKjC4UXSnti9Ic(@vif_RZ8zD+<^6=rgSVAz5$0$*Zpcw4TKhQWxj*#=V;HDWO7HcF{BojRP1LMePDm4yyD-c z!S?g+UmM5oHTB%sBT!fy-dd?O;mh`$?z9#SWXOu3^sd}P)%!(d!E?ppapO@N|uaI=x|A?g0;EXnSTP;89Q@*P# zAb2h^ijVsTxNT^_?0?`3Hnix)g0TNm!G%2$rf<{**!(LExFZ@kBSIo&;^wpuUdoJ| zdYN=ITOZkd85515^&O7Hu@2PhR`;u1 z6_cd8kkx6=h(+pVxK$Vr^C&{oyksTaN?zBuXB5PTyP5W(h04ESR>_lUCkN1W^ChQ0 zSpuYt-d4EMu&E(TT0fu2ExxIxG=Xw^D$n2gC>(ury~Etko^qWppXkzF+Q@7(+jf zbZtZr7!xg!&@?dWar-Sw0);ID2j6mdMR~bKWrhq79lpBugURq6=RF)_uZKUu+zmeI zck+Dso9MO4lZjlft~)D4$9>5lUc?dJ5t+7OB=Fx@w&Xu!*+2l6-IxLU7nV(cBD6gv zG$rsBOpB6@yZCwH=)VlSisObJ`aLfj5ffx#Q#S}EZ}sAqMkea>(TDpQ&pED3^xa$& z)7oljquKXnqELJn5(?DNk^M(}7G-4)SVx!r8=Au%@)A^Ah9%CS0S;iBU9GZ?ryxPz8QXjh!2SKRyMh;a9UpGAR+vr_aX_5vsd-$bfpI6fg_P*lA11Qm=8j zk$cNQ4E!@t8*=2xcB})$y*}lwVmr?fRc7#E@J=HJ_ckboWG+m!1oltNV|eiOc|1bf z200M$&7)SwiuYd{r%ECr3iE}Z(CYlkk?r1+%kB=guj;=0sJ9r87aO+ntZ}V?2W(%_ zE?r#sdq3>iWx;;GGd1vf#8gYx<8_L4;N3n zHE~H9?7o;eOVtH4Up+iZlockIZ3sC>W7MpoRpetHgzdZ`t5Dt~wmvaA1kQfy zAR7mJ@v&%RHyc~dTboPPJ>Uj4yjoKQe0*B=RFCtfR1xaTrOz&$?;3rXw5e15%D5jE%fjt*+4BUs3UOR z(F>Zo3r@5{T2s9^kE0~vh@SJYuRXZ|NlFqb8jAsa_qgmZtA&OZA_Gg7d9#5iT!))8 zZuJn+&xjxL+Ve=QQ9^bxd-U2-CoY06F(LcqE5y9ae@qt4X|>x&rNsN1N}11M9$D(d zj^#eLhNi3w)lZ6e9PPMHi51kkhQa<6?WCG4znBpJxxStLAb^|lf)S3+-a&d#qs{X3 z@Uh9Bwz5GZ1JuDHV~g#)2j^}gtSw1qshESQca7m%8Q*liVkiTK1G|%im~-L^YEMqOzJxr zIN?1DJMU+v%PDaqc*O}+tip0w7cZ*)1V~kjfwE$T)%>}jr4XkAsk9x3{Obfzv@N3V zmiA2%(Zq_W)E+CJD^G$#R_}ILMFSM$xmd82tFGDRzvELxP^Eb%J z#I_cmdhRlr$z9T?rB@xd3!7B9a|HBENiOw-C@+m?Hw90tgE;n{YAho0ebe$alWj14 zGukqpH@8#TE?I71^;m%L#JWs_wODF^7|Z|ZvC)xyQ8>Wd(Ib-Kt32WCEZK%|YtGsX zpScdlZZW-x)mv#c-YoA-uwZFvs!1ki$=V!cEYQ!d`8gUl&$!ppG9${$P^*<}RdD_) zJ*tj|m=r9y6=T`DGYt6s&{Q$!M*1b$w~#bbEeYvkI7st^?1U?83^H z?4s#(_aYl`+XisE<#d-#t|n9SHP!%mk zcLCNf5?mqHYO$n&mY?}nGR@DOQBm|MhXgB7U$LX84qN~SlrBC1p*$2KIYeBk?; zJJWH=Nhwr<9vZa9`?_<}fV=%D`M7c9@}-#Cal zqPI5D(~2E76v?EBdFRDuqGi;9WRV%iSd~D=Iw?LlovV)z4?h$$J*^kbk!C16IBrX4 z5-hL>>O44UtF^j?U(ft{xqyK+>dNON)Q>p4gc+IDyM)OKkA=+@P{)LJVtxuYk4Z)m zm6eycD|-f)G*MR;46!q#$Bh+P)|=^08TdIFB1ZumVDuu#aTku{-c@ks!TJh2aM2@_ zhivPOK?>xsDd92vp(~ltd*>z?aN@KbZTEf7HUmlwuQsLAOzBf(&TE2Sl1dRn?a!W3 zatw7$WNdW}f#+{}MQa7obwy{8BgwZbh$bzq*0cOFm!Wz48HnA!z9E{!xf3{f0vYYL ziygxa@}jIrm^@2aMzgsPro0518heZ1Y>Ru-6zJKR2wB-1{BLKr#wotYP!#wxe0<2I`#SvYyJx9GB#UL2H#1oEYe#rANOVuNWzi-d9(I-kSgENkQ_vd z$`UMCdwl;fXaOz?6Pog5p&}^y@G1svWg3VbNrDNZNtF^gK6{1W`xTtw{0bEHaY(%; zgIyxEUgd8J6St`2bP`BaXIrOS&HG!*R>t~|Qg%8S8BrByeN`Um`ID^%AFz%F^F|gU zyC0ad5RNrKs{X|=*`d1ygOlffK38OT=#1=Vh|ZzMd$G9&v}+Hxf203M22CVoAFVfF z(E`z4@)O-yOZAgRBbIKR<3g>b4zqEdn7LWwmq)g-+>$5S&{2)%gXFjilt?AI%^CkT zOHU|v6~(KmR`ViI64lx&O=@YCDNd|_ayO|^kZ9gA!b2-AY(DX^IR|UvB)TPR>hGFS zc^H^{rvizQc`T_+`$xT2OoWRJgboZxAa2<@?)nr~K_ZL7%e2%0wK;(BS2I{tmqrn_ zmt~F=wvnIRoL5*!0Is_xWc#NOmP3nzhufJI8i*5qY_4gsi@UoyPV7LIxj}@?SNr`I z11$r@sCpr{CX|y1=Tf@#u7Q3puf~E1o}Yu~cM(mrp8z}EGmi$R=eKg(PlG{Jiz6X@ zpv?qirqTP{dC*CRn)oljVGek_70{1End|)m*>zX!JfUGF#*m zU7@L}iGt0G2gHYmz3w+}zK;t%o9WGq|Ds7F@&)`iMH=q@UyAer%_ZRxK#?~1A3?lG z`50>I>u z(8HjdQINSsP^D{?@BcH12O7m0@csWr@uq)6@$i43_=);qRj}VE-t2EEUT6Xb*~zB| z+N<&ko&ASa6#giz91tpjSbzcEYQVHq0oF% znFd+Udh&#QW*Ew-`2OAq+{E*opfwYN+vwP~ zZQFJ7-}`-@ckeOIIbZJjc8?lWcdh!(Yh80LNpjaW{5LIpWJXXP3N4^O0ii#wlJCOS zv+Yn`>Rd_M#Sd8Ih9H$TuE#ikOrR^160S&Xb;1mK@9|8e6CZ1Pml1i8aB#0i`RiQJ zrO98Q)$ybbA))ewG}T}KgHOx+r?Mf!`K?y(C}i;e2cQ1^-+X!j>OcAPKgtHhdEyPl z!)K|ZMKcPGK`O`A z!o}~1nI^)u3^mdtIg~LIaVq8iVb$dt7s71+`%nLVF{(35rs5N?jorxOGYN zpV~1>hr-Wgv~N`OD)VTY4?HLDsnYyZ4H`-O`oZ>e-UxMVs_6>-hwU3$R<(zC*VORT zztmiaLt#cx!_D-14V-{eE@~>2D0f0uD(V!)X}Jv65)qAI3LE$i6}=R3y4V`R<=Rqr zs0-|pFFYhgI(NNi0L9E;!ZI(C<>*!>YTY4U?zk?7S~g8amO=#AS%3|X&CpEp2^+y^ zVusnVb4Ra{DV@W2je>Xb6{kZg5Ki5`Q4gD181R6U4*i=uj?7L721|--L;!21WuKzS zz0&IU`A+(_ooX2h&eYnH$74m8IA`GaWhk?coP%PBw0>V6AFy>1sK-MBMbrn`%Wxlu zJQhlUUA0ArM1N~1#raSh)N-nHAKn|-Z0xy|i0PZ8e5TpVb+zXzH?NmXU;JLd)eur$ zmG&?I=$VcwI9IdT#KXpC#Ps@yt3aIZrpJXTiw^OHddrz2{pfYNn|+RnFZP>Dk)$jy zoSF`98Zj-nHoOoH19aax2j6bpW5;Zw6Vy-_gbLK5`*Jtirotj2igP=;^EnRlMcgb@ zY?eW~I3mV-g_TgM`&FUN*2Zo!-2IeftZD0(R_!{(&XJz_z0ZSSki-cH?i`-pPgUwT zNk$RPo@Ho&2_%8ElfW;GD=%rr(ZUJiW8mzZwXhdcu&%cOZV zh6i)=a{0J#q}5O5;k!BHfk8<*spen?o`^qUGmN)N{k}BlpEswj<3{x5CL{@q*^3>Y*DM| z+w=q2$A!}82VqeT@k~F>FeM!NXGvIpruO$8)z&hxZ$#7?$h!@9*0FB{HSLD946Voy zy_ezOcnb_I>07}KE!CRetgK*1R^_Ry8CjN2ar`p3f-53*v9K&%3)@iE3b3ug{Pm14#ao48`%2-WmHEXU_or(rLA9IrtXCa6&!s zZi>(Fx&DQi%k|-nkJBb-flDIX2cT0JA_@Imhx^r1)z@&r z?mRJ|rHf(O*X@}wG(QUo_^~cj4OkFQB;hL%BHbk>j54zon6SW?uM5{y|44k*<7)^&r1>WI_3E>@vb5A+Qxw!PJ2rsvC>tDDdo z?oKG1`N_{$lXW5JJnJdgBw1!>MmC;}G!Lb>9`_f0f;yVay1I<*fFiHJ)wRNDEQbwq z{M)dA{r@#AIQ@}pHnCr_CG`@mHB^jeEnElZH2N)^W)p+(%Zg^di3Dv#?okIJwdLxT zme`P#t!!7Mc*IdX6Y-AnuOa^Zvn2Rr_0eXFCPo$Bb2Hp>a(W_Yj<>?_+?HFb4*@!* z*|u5bZZn(O4-2+vL&uLB2u0>-`{F3K_IK-{Wv(GLdUBn+sfGwDT9cC6$aG4}K+zKu zQ~80JL(IoW$rAfd)iq5CFw${WrJJyM>t#oDZFtlNpV)drJ`cSc4X43Ag^2I-y4nKm>f12ZA|TY) zj3w_{g8xG2F$NM2PI2CyF`m1Jm$HXOkkQNgH zmnj3;FZ5(>;k-z_P53jG$fV2m@4_$shn;aI9zOqofNn2GMzQz)FT#&;tcPLo{%$W@ zUvP$y1mFv|*d3@n;<$cU;*b(Ef2ibyh{WzHC1^;r-fTf;QJ{qAYayIZs7?}JW(X!i zvk!#{MhKJ(V#dFbFujP5y@BoYU5N|=M)uzIXlg1d4eCEL+;s(NTdyW#&z&x?jQyWM zg-eeGrQH=o@{aJ@w3agj-#yRQ@lQnDpWGy#e*I|^T_U%=gBD2uJ?Zx2!!^QyiFEy$ z2}PWLg}{*)W10#kdmC(0Ie!Z(u;*(%hW@(s=7!d^eTa~v)YD#lNd1EB_@38832})+>(a8;^t!9XI;|ZbunrGryLlpmAFK;CH z7+%w+O5dQ>awU(UBMSwww%sLEW#AB=_~ah5*A*Bm`qu!VKp+7>p{YT+y@X}Hlgzqq z`4&kRa|E7h7R6apAdcdg|JsRS(wQV0_P>JhU_E2j*Oair*P6niw3<}`61g`a^bD<~ApYq25F^c> zL=$sn=DPMYH(Y(SP7<_I790N)$6kc;Re=&Ea6lq?c{Ah0S*Aa&l@jsiZsDX4SMd6$ z4kIqC^WU4DZ{jH#=W6BTnuWy!@VQwNl}^<3Rl2t(^~656{;}hWOWWwwQVa?;`6ql0 z<}2~S&J^}%eb|;fF{TrnF3Xr!Rj-^*tUV z9ToMqY82_jZ;X;*Asp33zOD#EK(Mrw;SFli`IvNx3_1>&SA>CA6)m|N@ ztszyecK%oMylri)8QY`$Or&T_xM`JRHp; zvA<>^%%=ZQn0FNrfLT>s{^(lu>}Y+mt9eZ^KU2Wgj6_kTBIq36s*`2+F${Qz<&ReKf!BxkBZ9d3A)Ld5fTV386U*V0jrhKeDZ|ocZ`w zRi{_c+-8ZZnZa4V#8s>MB0@U7qDU?VzxTwB&}mJJAhnhV846`c;xE!_M!;lD;_0{G z#54PMG`))iBWzHI1OY18n(r95F@KvEhYKTBNwvA5F@sR~^gaC}{+N2dD>UJ| zWB3ewcWk#MYx+P-0BS|n%*E^M!joJ{?1#@me%MO%?Z}^x5J!WJU%U5|uRBV|q?IZ! z!=bv!1G1R$uHx4RTAEv%pkSj6w=owG8sV`zFW07))qiN=rF!nul(SLHPxm`c*0oM` z_4ejrJkNZ=;kz`%x1h`A?+!!f;#u!+${o%Yi_0ALtamy)S~EW|4XwCCd}2B{(VY?nV}j(K!||o~ zn*(Fo2Xp;m!1=IWQspsUAn~z0QDKp<3=D1x{>Fgw)5j3-i|$KcLYfmABWKh8W=3Sj zQ4NVr+q^&)N!^^FCu-~IvWgmG2r^hh+YfNnDmr3Lgm9)%Sd{O&AZb=8PiOefv9J~AsNR9?PKiOlroTQx9!pZw1V_3M^Qu_%yEzcotxAR{uRF)| z&pGQD-sXz3b%Mmmn%>32%Nel)m@-GN=ME@`S+lHAm!-VfX2!bC`i0RJQGUIgQ3{kBU+1f;sqvSl+&h7Hqicrxcqe zO=1-}@Z6LwqmRZJhjRDIyN`dh1bIUaA%)SQIOY^3%5lK9pg1w7PjU(=lQ zy)EDFMU+i^b^8<))iWk+q(J55<6ic9<9c-`Q3(0*5)D({<@*S{eW>7_Jcc`L!HO6> z3E^K1WF2Fq@(R(y8KWZln2TVnGFx>iqFN!W992V8NC zPd7UEEJ|o}D$l)HoJcB6)RSSQw1fTK%Kg2qDmIH|e*2A=X1^iSxF@>2qB9J~8N^x$JA{RurAhAW0HF1a1omj)Yu6nxD3GW#h_LC*x7 zgccs$BX)_0qJ=M+BFlwnJ>Uz%NLIi98Pnx$9)3lVKeRZb8kNl>yIT>2kPnktOgS#< zf6;Eb%StK3hpMvsktT66+sTQOQ@_`*EyWUfFjA1GTT#Eu8^Rbxx~2|=7P=4&#fAMA z3lEIqWSsrz74eiO?)CujlqZ?bOc+7hI^qn2p8qAZ{+1{bfL_B=Hn96Q)9WUx;frgU zFZKKT_8^NCEQo}x+{~RlF$_zh#ZDi_LxDD$9Z|xG5EHSa#U9ExpvA^M^Jp+5-9j5G zG?2!Q2#fqBJiIR}jh&`N5UC<8WmbvZu3w4GHZWAKAndh0xw=$dhuEQEdNZE%zPY(y z8|Teu4bOX$4bJM#CpSsEG%{^!16HAqmd>tP$J107F4{Om{kzg|h`Kn>o6(yOzQCX^ z%5DO4m^m!cPI`D>_rt5V3_EW~D4B#6omqzwy&l`ph^Qj3`})Yrl4$lbPR242#hwEpmaD*ZY#8h> z*`ife0&`$(WjmKIzk7m;V0(@V4LMaFsea32yqMnc53<&ABGCBN_YbjF!(6&o1q z?D;42_y3dm^QQit`HNgu+5zG26?^n8mH*28?8B|c!m(`sL902PW4|-MpC@~9ENACA zSamNF^r@^4SMVPUBo61eA4m6JlW+p`;`)39!9MYf^`ys;3ti6Ve^RP)bZIrpvDxeN z#e#Zx%MuUq%9`e#K~5RdW*}#S?eOSx1+pNsy_=7qXgAAJ4(mJMEloHzg;Js%fJ-Dn zzOa^y9OVbb8dXq&%FZ;z_hp()p%i*&DO3W7)Zmv9Br4u}#_j_Aa-Gy#}y{@6pYZER?FO{Y7nqK@78A=DifS7g5rE+Nx|rnC7wfq- zcFJ)Xk{ccL9sIJFUE8&U;Fb#mCa)SPvwL+>wnOaooFzk{P5v$OL5$*a(x1vn7;ejq zVyy2a(X*=69k3BH#j4J8_yFm6B*+5kZM#15?pL0^;FBNLH;d%ECPN!RqSp44?H(qg z<4=t;DDqq(Kxj>IE}X9Qh|H#zcU|I0fTT@Jt8Sh3CD%fjSMTa?W&Xzaq?CtOx5v(9 zGJ0QSE=DMF4)o!`w>>iX-R3Fs+Wp<;sqs&nr%>6|Pc_;1b2ZtbHk_(PfkyhO#E4&A z+{`5MoQ8C+K+=gg+5c2Y23+5hQJ{PN58nqIg6x;ph}$hb^!IYjyjh;0sgF_*W1lP} z>VA(1`L~$k_0zgE3XkJ8?m?@Lt}aAOg*s0s>`|0^0YgS_FVDG9Q}}W`??1~V_|2us z$v)k#0CGm#LGyYQ098H&=k?0M{PA_zG0cfy<NHxzfxJfNo==rXP>15gwdgG=;c$ zGcP7CK3D(fmcES*4;Fss2EV79quKg5?GFoxzRSz9+1zj>&ha&M@)I-)EC4e zpVvJI3~*&AcVGYpbpBH!C7TmHynlY}eszDeY+qvlE(!21dH+i}$GsrH<@@p3j75h6 zLQm+iq{mrfc*sr~nD0*kglsA}XXKKwhIGSDyg{zKpycu+dP~=Uo-yq!)9U*m+karS zJv%orMnIK=bS6q{tkrk)xBYag2RryGcqlHLrbpUhgaJBe56>BitEWx5V6WIXki#qF zx;v0iJ2dp8{~uN45aAn1yW@O^{L;sari@Es2@n9m0fs*m#*7o1IMBC!)9$$hqu|rZ ze)+O6`1A->JB4&t70Ea9MfKie|537d7Nx#JyoKg{KC8$56$0^TVE2K50RH=9TNLsz z@A+~G;k!>2%)S@2qZCHA3{~>KU7-KFji6l&f-=zx1S<$3U3xw@Zkz7#j|!9ywR@kS zYx7T8=wf|l5?EoRGz|lZzT71r&;4IgkP!ERJBwM4jo12*EQ$8RYA}#wWOdFzRq`8$ z*R~rjLY)z*H-vsl1MPPKMSLrX(l?F$@*E5#=;9;>+_O1KtBQZ<7hk2Sy%uh% zZ_DQ6I-qY;Y8!H6I1MpEop!(m=)Y^gh;z^N!_y&Pk~~7y*$GAr$FW>B(E*+G_xLy* zYYcQVNGJIaq6}t;gi~Uw?_Ecz4gVKfcii86trE0|;`FmK&un#eT9$lvf>@cvlOY3} z(iq!e6>d&ClJTqtWGEP6bJux|jA!k6fn`jplIg5QB19Ryx7uUPs#j9vf8r?Z6T|bF zy(B;7hZUM{$iiC(2J%umJ9^o_c9bZhZR!TRXYx)z5b(sWK!u;3O1b&Vv-*E3>MZ5? zr@#8t_UL5-5Z?QLumHA2XDR{nx_Z7qX>zAIrhN4sNp}kEt^C65!a_auF@v>MVZ2b< zDWqSKtJgEB6?0ZFvM0iH3`}?mV@>GAW08Z)* zV5X~=@U@LYdjjYuDF9p(Z^VE30>JqG=~l zRNsG@)c}-s-zoikKBEPqzP8V=A7S1A`ZoFxfUH1?L4{=4vbpT)5?6#zW*Gp=5t)0p zm{+02ms@E7^BS=*wczX689vNGk=QGFQ}=DGHVCZ!&G`~AWhCt7NpE?lY%JtUS>UC+znU{3M>aTLU><#4}cYZ?2EL=ZJof! zeL$w?w_$S3Y!mQhgY|iw?xq4GurLAS1-vex{eD7lxyHj_!`-__tZbyZutlXilIZ@o zy4EvzdK?aFmaEc^D4FTc0d#|sO8Vl5kJZIqZ$z_dl81tkUN09Q=kv=z0f6uvBKG~n zlr8}je0=0T@utDoPo#Kn2ZAsXfu*3Ln|5|e_Bv9Z=!0i7Z^XntGff(fpbQT=4Wcwz z2SF#BC`QYtT~}j5Nv?1B5{?-txkvM&+P%Zy@ z_=Gbn+7$k#!MAdXFH;yuxhP%7$yI?X3FD|5r8h?6ip`ibF+t zv|xwzk&hDtPyxM#7kDl@fNlcN|49-6xFT)r_n8$sT=a8j1x1 zR_#la^XHTE!|~Ifh|$EBM+`od@yt&W;_M>FC?(nhA6BdNSf2X^QDLPvq&oMwBlNkC z@647Iq{rj3O>6GrJ)j=PUyts*g$@XLS?Q2brj+DaEx%_kW6u~?`~qZs_Sp4Gt;h2$ z&_|n)J-M<(LcL>Frn{rJ9hSoDyhR(Bn1AXZRhg7`$uNx=Ehha?fbtcy2ynPB7tdZT zD$#2DoMjb+yS_~g3qpmTaP)9ZH4f?aw|kx4iCF7%y>^K{4=|i6k#jOMFOc;`a7BQ6 ztLM{d^8nBZd|w1?Qz^I?&I38kinGS?(Z_jLnB`QtD$#NCRu{3kGJfH$5-VoZy{b@O zE;n~p7*s6E>WJ~VM$5Q&oGeLZIB)tpd-RRpYav8E@(<(DPflLO04<*ayn9*!EV#Is zfE;e9#kU@PXLJ?`N*n1F6_9(sS(ppLr=cL^Z5t=9tc}dPk%c_+g(l7`U8ne8Kia=w zk6fbcVF?<7V$^zvh`h08zF{Ir)%P5 zKR9I5V!dP>TOX(HbrMr6`^o6C_1DMjW31o@PmeQf*UZ^=FAl1KW{hwBm=%;lXiYTX zWc)*G&qET^Dly?e-+E_2^@23oQY{lOl3!PqgMyFYG_y>{-Z}#@9&Wk4$@`|@IP)2r zWN?ms3yjOdz+mL7?MUhydJ)$BeMTUqBBWi3+?X0JnDf2u!`q6>^09R3__xZY(;mfN zCvs}bGapfw!!24E;2N?-#bY-9*&GcUkSt6Rq0C zJ%*Kz5WlJQ-8B9T|NiJkupKmPrzN~epYawkY{lAks3OdW(1(^N6QqTJxClSeTv^H= z(tcpF5Pdwn1T6?tyyAW`c1T1_HarJ9le&$qP2Z^yvBVXj$d}bqkw&AG`t7I4qebfs z7{FdSh-4=8MI{u0B*4vL(iK7Gq374sQ6uMG#FlG~3xxdHw=+p@7?(lqyqCt5;q`R=uDiJ$ zlKgU=(8!P@dcDXF8sPMDZEM@O1F(Mbxgw{H0xZb(cYr`g$^^0@GUlj5fYQZk;w*KuxXQfP!au~$yFyql|1F3$f#A&Y6+s^rzVuSe%wNBG1e*!g`$;&AYv9wUno7U@-#mXo)q5?%V2B0Uf%T`&pX^Z%asf_IO3y~Xm8d6IIwpCE@6yXn$_5@A@ z{D=jEv>UZ2B$`(FfhEAQ0;C!QpUdPq(`<_^sV`DBVJu9Q3DlBT2puI4A|jB%OCuj9 z;R~L$A!d?(k*W%tBWuT_q>8G(E9N3Hilp&9j)qpLGN^-*(XIQHyM;;i5xZO=Ol5p1 zX~`T356363QOhZRjxj+LSE~r|8XJ>+&}V$}qfiL$3+L+k8smHA-0lQ8Q{+uX(MxRt z65V4MwtyF}YR$8KM!#fVFBK4u_lhCxWUFeU6t-V)A9-MY!m;%bmQUgdH%-jJd{Y$| z3*AC;46vYx=vWSIYf%ly-gVSlC+3&wk}3OVTNC8pms>LDaWY#E-BmgEBUPQbw&9_! za#<6s#XT`J9;AgNO4dm-xsR}8$^5rPI-E3;nVzo|9pN|#atDqQ`>#IXqbiQpChNk4 zZ1;e#n_O4G@Qwb;&wGFlXjPW14#l@XgKebqn};ebw&s^u{O!pnj4H?Y^2vwp+>*h( zbH5lbupW0klaNWw%82Pg@-n`tJ1gP?!yUI_WPO}e31{T0!;EMQOb9rLwvZTVRZNyB z2edmz?Bl_GgGFlvz{Tt0!$iu&&M$OlhkS@xAJx(``jq2WhItCu}m@kt<#PNHFh+M!GjbF9+r#FHXw(#2?;u3Jet(0LNoGQUDT_ zyI;%5_l9LV&_|#LG61&XjgM|JFaydo;H)d+1R?-!s~x70D!igj4g2OZs=x%A1@cSc zv0h*ZX*L$`AZ+N{*B6S)I0xlQI~X+%iS(rWo#aIZ0H<#kp!Z?p!18x+oD}zw_t$k> zzS~X^?70S(&ZWBnb7!;lg{Tqx?lseedz&xhT-dKMr;y@j7n!Tnd(wmdicj~rl*d(S zOE{G4mId^BS34gtwAqj#B*aCGC{Oh6bl!Y-0Q1MQf%NxLP*I6+fY@gk=pE$OaHSzImrc55_c<(vz!E*}pgwv(&$7Nbgj3Aw@ui`UIRmdcX*l6c z6luT#>38vaJfnYbo#1{wwON|1(V`PEr4yYHf_o=^F|mw^22x3t|!Qi!VY8sP-8P%m#iIoz>AKH;$A zCfx-%@{RmDj;cJm?>+V>I{Dt0Q7zpkns?qRh4njxFv3_@WnUC(Q*kq*v9F;C-O1OV6gntUDxv3Fa3 z9*WZQ^8l~cmBVn7xD9>!2t2P(Z(ah<4_v+$k3P)BQ-msc&^xzCxU`y{#XfMoZ8?CNnI zXmi{cEg)l}x={ok2}+DfL&^Y#I1J&5X*8uZmFtE?M$Mf;ouSxhl>sl(Z!yccCb z==VM%b6@bO?XNx@v{QiofbLa5M?&qxM~+)eIM+4FEalXP+jP&wd-FNQDC&3Fwe#Rc{h>lNUeQho4(K_?GIC9 zfs{H`Iz};8BAx4&Sz~?ATgh${X*!H>*uP@~Ud>3$FNrJnV^e;UVQ=VeO__21l2ds| zzxLr<;pd^v24G%Yraf<8ktX)1_d!L0uat~FWOht(dX3&Si3n@pE|^s_)Q?)k6-9y` zP65aK6|wbsPrsE}8Z~stkiR^H3=_Eg-jhw|2|6VvXG|-u@H=R_zlnf=$|OdYx|db% zpj24E!|EG+2CN(FxXg7X3(on55Wa_-6?3#}7Tw#JnsCxpMrmm-))z_rY&3*0wZ^OG zH?9ag5G`31%28&a+$fDh{SlrP&I_!%KX;I(VPd(n6m;+edw5=IsL8rO9H9b3rd{pH zUymCvkwt7a#(;Do*i+mKHeSb95LoPqw(5!bu72reTA>?i^{VRNXH<@!LYu!~Av>8o zjg@Xlzrx1hmNW%+O3NAEQ$dxxr>UsJw;AYhmFw6*L5vCkToPseO55J2@ue)xQnll` zMZF(lQgy-{B?E9I>Twrs8>;7Eozo>l}jGR)TI zk(Ykj#|t~&X#93>o$S%7@3v2jsS!+FD(h=vuI4OtxJud~W~iGL(xj{th+EtRuQ>peLs0cI+enBw-(+cQ3_LihV__KZAMN4RvBBR9_k`2(%}^y zb|3u%2IZ8(6pcWwF%Oac418kKTpG7cxy#|Z;oz;9HNh5t!p$TJnIbLlS8j|4Mu4tA zfk}I^Spxy6QnkI+6IgM`BtwC?xynGYYUZ&=WE2P>O6ZL-igsgR4DQn<}vlj*_{N3Sd>dTyjH=kLY9 z$937_DGY^<`%pW-j=G-=yG1J}m`ZA}DhsQAG?HALTk3b#Ny*)ih3-kBoX%{9bMG@` zxGz!m1F*=h4P=N~&F(WF1Wk*T|M$K-HoO~tx zv^f!lD(D0Qhm3f4;7x(Gv-_%6%x%X^ODy^jwEX2wqpob8r>)vBf z)^RGY;t!)4MszqtlHS1p%4EVw3oYo;;*ZA$F%#sCutwV#wavWWGJEuJ1N`IweR#vX zL;wy-i0%-&-eya1Q{|z~xf!C0uuqntotWR%wd0Bu1WJW^1gD8r0YXt<3N&HYWFJ_EX;ozGB+d+0%@TTK26&Cyd#X8oe@tA*bf@ z@&Ubyfm4Ln|m?8(KAN$OU%e4DulcaM(*>%?-9{h@svA9 z{&GDX^NP{3zKIvwQQ!?)Zrb__?5dN{X|JopQiY-Jdz54G+>HooB&MmnZkAp^??g#2ed+wQgkJGEijn2CDTaF;kfLjnYZqI|p?%^xU6 z)CNoW?k23a9#9GQa%*5hfYr~}vIREXiYQfOJ||riKRr3}RDHWm<0ixeYJ#Q{HJq9c z*Z|e>L=K%z6{Z#4c3oLsOjD*t;s=Qwh>omp? z_EN;RuNVs@^jW$9Je{ALLem&KN@GCA+I5GV@tv|Nf)-0HYITAhw_-QtX1PHRO70%#nODEMJR2i+Gax_wKd4!)moD9Jr92vM$DPzUQMKN(Gj%*nxn5 zdLNkuB|H!r>NXSnk)os?$d_eYV!(Ex#f7P;txelxWYJhkpmVz*AxVn038 zqN_8@dW97{h9lEv#3b;QB(vFd=+G$TNhxLv{a7x{c?{`wnyz108C*#Khd2*6SGIu; z?qX}S2v3-19sIagQC6VpeuuqW)-EInKTo<$3!Ggp+vw5aBq=$%Jj%!Lh48EqC>FcmZKsRJv2`xWI5!gQ=@rx{iT0&|1|r2`swZ0PYU9 z3tD3;n@p=jWVvxSUPx=q$z+!9*fV?TRe^hg5CH?^+#EjQEu2(?b`Z(3Qg{Nn(GkKt z>s_2wO@!=CQ@EbJ3a$%YDf7ILSG+~mZhE;=$V8d_uV%f@wCjxh63qjdwle4dXx)M+ z*;`gBx+arbcd&|h>w=FGr5a!LW_eYw%_GPgtde=ybPsI{;wI|B$;xIuuO2THE?xBD zw#88AfFXEuSx81EdSt4);|F*BU>dV06;+e-oo#OOVoOWL(ek2k%e<&sY6N2E*lb+_ z<*n!%CeYZf+2%%NybaO1nY?D(dFN6l*l5;GxYdt}W_+7vi!%=jW~U=G*XA>tO~3a1 z-w{`AKEk{NV<|w&hv@k@@QQRD%%J$&7_%#5ak>n%t=>)YGVLp4OY`01}E=d}^rYk}B0lxR^;OBk&*@UXz$LwB%6%0Xt$Z|PP`^~I#yK1f`yxeSBx4T`b(X` z&JX%yP$g5j#f5aASk9--IGrBy(<3_MT^lifP8B`5MW!@vr(?5AVKK}j;gb^}IaW`{ z%KY*~5+bTM_{&zF)`-T4EvktyxBjnaf@(*wweT?K6xYe{q6vuOk-^9Ws+bD$skJQR806hZ~$04vfaz7j(7f#z4-hX;xg-0)A>$v z(5rY|_e>9NM7opv?8jv~>jw9T+rFV1y~V65^d8qv=~B`9$MrsvuLfx6a%ucRQ4><5 zNtZ7)j$w zV|RniXHML&CdICvP}0Q7w%OMs}R&pwgctoSQ&9yoanFOdWkr`cD8F*B_-F|FGG z&DAzXYr4K4Z9RCWs!6jNSAPADQTD(@!Lf;b<$#U0*fga9o*FBH&hZ$37F z+ET!~xQF;ZJ0h;cZ~QrgUYZ;^R-XtkHA<`J(d+}(FYSl#ylA5|(zF$PG8ZhKembn8 zvYt}dKtA9#Iw`s!8O*4h^+0x)BwLg2Yo(im7^9oo411UduX}H(4WxahteLX02?LZa z6$d5445MVmQ|AcRb0T?}wtk^6DHFF*Fm=!QV+SgzS(PF3O>PyeCWF>SoCSs(_67$X z2sU>}1Y23fNVJ4CuMD`NA&pyhc97ftxX^4l z@dGpP1zYJQoMYU@2jwtARqw&_O)|!S$G)2Z9+bn{Mv@lmat!}vZvwZ^hJ-ra_mdWqD`nQ->AJJH@)DI zmKAXCoC;8**!dPWuk~Skq10fCIwlf`PCf!!mkFrQwv8}uE$L%ZwwPA5`3`>r+1Iziw{yYihr^SGKo*p7(S>=E?u>z>H`|;vWp??=L}$T4(RlkU2RI?y$Pf zCF&FYcD+b2rl=x;o|#My?4fe|PtXy;_@11>!{WBK-=~4X51IL_5Cr;g27JAkt+XJ( z&+Hu9GW+r(AoB=n-(+4m-O($I5xpf!=KS;^$4g^8eW7oh&FSR^p>$X<#_&Sj)n@xk z|IgNVY&*KNSt+)V)1~1*e)6*NV!b-s|CgT}=-jmMZ#h}_PId|p$N2EeX_oq(8h6)3 zbu|m@tq0%bdpcW|5wDXrHpRZM>HtWajbxhfJBMBTgxHr27rfSQ^Wo|%_}co9O41b2 z!{gUwmg#f|XXq$gHI;Y!f+wRd>M45hE7G@)P)qEdNNo06{H@9=ER|;sHhPEWuf(&r zExF0j>Ix6JF8|uoJIa*iOO~plFiU(3Fv(SMc1|+t2%Rk2!1*7&cS$(bSF>^OCGTAQ z#~z_N2c9DINz1Q;?XuN>Toh{*(;lHHrLgF-V5}8p!%u@8{UV@e+NzPH=8p(uZPxNQ zossp7T`iOiBKlMuXz2b#3gZq~2YWeLh`Z5?R5R#ro+e-eK}9|}DG^c9 z9p8AFLg1G?$F^`hO6|$l%^>w~k%$@}_cy?Vn&uDiF=F~|F(%pw)g@#{UJ>rFQ!2wj z<)}5^;UY|jSmeC}mC5VH2)zc0mJc@S{Alg$ykO>nZ@ockh`UIlFYbB)+l42~`W{OW z>e9z1S3VP*)qQ~j@jXFxYqD>mF>-ws9H%TjMtfvXqG>%8;x2ZQ)YitkWMA!$Y;Z4; zCr7K}R}4+OGSHEO<=462^E$liBQr2brZ=g(WD>U6(5cP^896UtwxF09v= zTN@%q#G^Jpg$g?qPF+y>Fx$+CnHQ7SFZ4aVLJn8DJBqi;w3^J69ot`1oOMuBt`-RL|$f=O3S$37%LL(ovp{wE47`k=&0 ztz9Il=dmCYiid!G-8$0VoR8K^S)+<|mKUcxcq-YYsa4rvLh29)UOHVhW>2Vg*QxH1dU zPH-M$4cnLpr?@S+;xDL13nMgD4EB+F6L_o*(mlqx_gzFw8HjrOvW%E6(a*F&2Ir`* zfX&Es__%|eQEY;##7{K$PdXwmERFO&m2l3MBe_a4j4BH7N}ClgY2|3@HEwlJ<(r6zf-nP!W6lIzg$9u2U(%4G_w)4h3aDBPU!_wYz z?iK*Sf_x5Ck4$OwO`s#pj8cBXU8paBmG|YVW@{54-0S^LXk}BEK}j-=yQbUndp+~H ztD{Wz$FPKkWgDl*ak#kHz;se|pvJ1p!kW(>pc`)#Ck(P1L1L)^6W_m{zrh25MI(jB z;=l+Y*yC#vKoRR^Y8X`f1brskebbKH7SBOP#nSU|N{xS9j1&xrL}J<37v}3TCJA|B zSsTn?ob}Pz7AFO*jR`4oT!qZV=1I^Xc;xa}QJ&6#y!s?w*Hp-YlW@_zPfvWrV0<== zTYcqx9SR%+RtN5tGXWIb!yR#-6!f23@)c9SjX9jYr~og>RfG%mwTz3qnO`VR>L=T+!q)|-G)p=#CwL{W6tRLB)d z=EVevBW768)O3EJHuX;J;N-e8OK=A01YRv2QkZfWatx1aKhZsSN3b;`sGMS zFh4hPX&*IxEkUW#Xl`5RU`R%IYf`(uL@V;{dtRw>=$R~N2XD?6p}6u~E&@6(He<@s zJE&=)Rmc!d2=W;a!~rNPUw5VG(cJ68Ovuyz1A`=!G+V=n$7|K=b&XeR>LoiAr48kT zh6o}F*7hS%X;A!>*5nrnQ4$G{=MR+zV~hVw-nRWj~ot0H*Gss*i<+A9_P3uThyo`Y?m_BhGP;pwEz5%(?4u9!1UV4S{Bi#r3UM7Nn2 z($Q`#BF@A9rj(Gwd!vFi!7pcmrb;F^M@d$r*0W;Fe&JAT*tQKwSs3zZIWq3Y_npG& zFSC|;(`U;c53-J*V^QXASfg$5GZ;G z=+Mv>t}E<3I|GFxcV1ATsRGH>^N;-^B|%V9Y1hi(AA%hcwmt~a(S}TnlFZblBint-W_M z80L3|omiq222Jtz=f7H-)Z<=+8Rh^Bq|ZkemtS-{y&NrZ3xjahTd4BTitgvP&PbNeyhj_~%#PD;4m z9-YqVuoXC*W5@e*K#jFqJTd0X(UlFQ%}bPI5hU|SJG|2^xY1_vZVhX0g+%{!DgeJz zCwX|SP*PRS8m*roSv_ZrrYU5m)4awF_~Wtk?*2b?y<>1E-4`t!+xEnm*vZ76*tTuk zwr$(CZ95a&*6-%|zxCGrbgTPYcb)EYs{7R5Yp=C7zRPH3a7qSnZ~C_FVV&Q@iqHW?H|RNJ07(o%Mg6=iyeqCvQ-(3zSoS2hAOTPPQFSGMh+ykL zL37eT%qe*{M|s|~e@(w2n5`znQ!W&7APum8VL|-+0LMHjidD?2tnzYn3ns>}*^j7u8pAp341>p_=-)qCZGBE;q!FN)n3B)KGp{Ujr|LcZGp z&WRRYPJwemmgMG6qQI3y7h2u`9mX5K9BD|0wA$1Q;~(?h(+|+{L9qrFXC%6(ATRwf zoVN>E+c&B3NU0?xJsitq(9~bIigbTa7>+ggLYW2dRk-h?c;YWAstbv?9vp)g!C-5T zYshaRIr+nkPnqsfer~z1R|h-0b#?8W5Zb(EUpI9sM}Ee zWuh4CLR|~ww7Yd10IiD;UA>X_cEeUo_J1arr@|5#Sp+&Lw`a7vB>s!pm)bb~3*YW} zzW}gbunSS!*EOa9K}JShUaqGg6T|lR8C!bbETv|YFhfHk262D)jV5vb4@-*`uI2(c zjEnzl9Mbng76~Os8pFekvOnkCA#W3KleNKw(+Ojih?<}`JA*y(46s-k`?dHrQ#=0W z2F41!NS1+9Ee+xBQ@T`sO+-lreJ2^2lmi_&H~A2u{XrkLezvjRdFIA)iPj< zCx_5C-qZ_zTp`sQBG9R-Lw`6cxen3SF@A0qUr>HYd%ojF0c<*4@1W^lX~Uo}=eaT6 z5!>rO0H~P%_%5(-f<6U#3F;2(@2RdjwP_S<^Jw&R{W}Zug7)O|FS-on@Zb%($qK&! zT6ZP@cspnPm>U~dkgWxf8YYkzS_H2NP>02oe_-CEv0lKW7ZE5$oaB0}Rt%MbEi`W5 zAVEWYq*7FXWkpYMjaEu>rNvPwBnP@ma2y`KJcS4ekqIMWA}M4oocwJHP{e9M7pOLe zD2sag-2B0BWIw|&?U5t0S@MCGu;)Ip`4F_-wIeer)K05C0tG2AcW5R8aEhwLC$7J1 zf@z#m(ZHw&P>Qvc4DdMx3b(!IUWicj;RN{y<(YMf&8<7cW?wMcE*AziG8*FuFcr37 z?Y7>$((tDeb2miYWhiSee~dsx?8dX8sO4|_%qn&yW#X{>Aq9=UL1JM(R&q-I-4mo| z%@qoCvatmT5=O((*~f3MvpGqS}v1VjmWQ z@~%At4N7abg@iOITXnVIgN4VsA8ahglv0=HG0ei=WSac;L_36jK*k`U-S_^9p3mpi z?E?+|npKV`g?$~)?`9&Jo7lDwOv!@wT(q0Cn0y z&Dona=jgUxOs7MPtEp>TM;OU-NLv=l$4Cg>LTAuNyQJGoc9zL_sx3Dlac@F%IO!L- z?DfX;KfFnCTe|DB<9)ok^eqo1I{z&qwPP+l4(;amU)HmvE+*Ut&iqS9kJgWB1nt@4 zv2g8)T?QmO=RlD55z;1Zak4d~E1!#GtrC=Hty z81T5!ukn&JOAuLZu9>(mk9vrI`ut-vvz$d9;?p<=Nm7WU<|yupv(ksEz;Y>>pqp!q zI7up`FrwWKEy!E77E8O+}t_1Mvh)L!hUj zCxU}!>Hsq{^o`HDzIN{ASH05Qj%&Ua|E)>$d}(x#HI8?$gpL=9T0L}&j>DN5=#3v- z7b)FJ-FXBooguG?@~^aQmQzv^?dJ)riMyo{+M=k(ZLYzxxSp@3P?bu6qKXrtRmpem zHh{CeiyPPM%NKR_^PSI_%ZX6R5)fMX<*QdE(7|W?zO@Ey@tssJ^5`>>w({P*lDqmY z8i9R*ZXI{M#$46c)#>_fXEJDmv8kQK%VpV8OG`;0bhjzWmO<8I-kii^2=VYK4C)q3xsKtNFB9rj4KGXwr zryl{iS=w2;CI`MV@9&>4qB)$80K1sxp4z7`>+_FaAiQAXj2%aDg}TYS54%Q`qUAqg zunAjZiwv7QNHW$f?NvI_sB;3!Y_R?nTAVr|Z0!6-iHp`4c{X6*Q#co zRGo5dlCy4+%XVbV%&C3K&7Sj>nq8fk{a2+}{mCN#R!%|Je5}YcmbvT|B)n`fs~@&l z;oedo*t~11`jqqT^NTgsqnORJ*kai91U+{cykx~oLe%}li`OEi`FzpK|Iez#%6HCs zrK@!@}J|%BDvMbW4)dNxl{o(I0?5u)k2Sb;oT zUkhptYwdAjLT@d5j~`ErsrH2%>aD0VT0$_xf6{5vG!OQmC70Mr95O3`OKQ`#hvG`UY;F(q!!5;ST5<(@` zDubxo?Q!4yct4&#%FB~_G88=e0=x)QP7!m?H}T&w=1i_p8RirLO&aT;$(LL834rd| zQ*18w8{Z5)9X-=GVDZ6!2GAXEngZ~J*YQ0=SS9!hd^)|U8r1Gwf7zba8EhvQflfX) zixm95&{5Y__ic83^c+u)H%;R42z3Oe)aLS^+uBa2C;Jm-f)$C zGAcQT_ZBlLW;u<0B_aA2@fJg5N?N@@pQmyXs93?-T-G26@8(jrCHlZMGfjd{qYx)( zZ;7Wsf1*E#ZP>gTe8ow9STkqwdRY4&bIOzZ9pIUWI|D@BLuoX7!sqKQ+F0V7PAc+7 zS@s$66!?2?SxK&I1gSZ$hKILy}WqS~_ zWm6Z6s!4pF%0@Y$;|`U<41>^B+$%1}`pX?`RGu@-+{bNh+$ID;o>;r=FOhG1@;X*;LzC?l1u6 zf7Ylb(4&b|hj23r(Jr>MweXVcfzBPS&{rsyXKQa zK71v07c!A=gDkVuAsDNnJ0PFZmNY-?679tDDF6D;8jO(X(suHlo(i@OW~-&PdZVYd z%MFkxGB)qyG}n;jgwx=FEGz^gQrs$w8qa zZX%%Ain$LOc#0gc{1-Dc@kU+>@BJgGTFF?okAudw3gCWNk1bq99iFwYM5H3;dMgN> zE|+yaMb~$E$l1zi8>h(JD$thNMBV0+#Vz4Mr=h@oTGZ-@q%DK_0eKjIz5JqAQ$Ok8 zWZ+E|F~7a`I({6Q=iF3x=KJfa(Hpx$r3;o+?3|lV!Bg03PG02PoKMoSJ?fR-kCH5j zXA*HxL#d?iNNsYHQm5;?(|PBjxd-Zr#9-e`4-RN_)}W&6?v;Xbee}m>$BPgP4e1+T5)7; zN6L(vdTY*hUYN-2WF(SR4WV&eqR-cIn{zBL?DN&+bk@PH*g{;x7nyn*3 zl&q3{)~aGx;G;*sU`}?XT{*FFQ627BSyA;6yvQ5!aAA0su3~?9luTF4^+pqod}`0! zmaq3AG}TujUojobPcX5 z#*BNScKf?m?pRaIvLmc>rGlo&j5#LkNOg^kR;PQYrE3UbL~!TE)cqrOz36u3=C!;9 z0GayraQU@+^8S4T&+~NoUqW!YDhfcr{c2^?=d!tpR4$7C9Avm3)}o;O$lDYo#7Aed z)hEZ+sX19?zw!C0-u&{ijCw`nb?!egdyCDRmg7>o?f}{dyQh}ZqLTw+29;+FqOyK+ z_TV>-)-zu>=#dc`d(6|>=432x$e0!q1X2yhF+y?gQKHRzA zje`vkEz(^&EIL-J zU3EoCeA+nlGUVv!)8Lqi+M<#E&7>OK{G8#={rg4*^@4`MBQVkEo1+nm!RUd-^~4Mc z5!4xw)H1F()H3W*=$JUvIDY7VCJbY9MGp8-G6M{;x%5&A0BO$O2bZ6hG@joFpH6uP zd~A10!@fz=ch|xJLtL+aK#UNH`6Udv_tjObEN+al#MUk5k$-*Q-O00EjISZag^8v@ zUKKBhdHDoskK|hxAl!y61Jl*Z;o`j!wDmv)y0E#aqIKVQ_nRai4=k%%iLl5;*W}rt zS3DXh?$T$z_vE~fNO?0$m7<>l0-QS|6v07Grv0$*b^{+GeKY`k_>WI-H`Cm#Pk;i> z6%#);&=y08E((lyB&LHJTu>27u}v^Y15`9=pLFv+FR^SC^kBHfhr)-v--hu*G>=ct zu$pWZ&I5d7phMj6iS}SuKLLqsHs6Nz?H~Y>n3cIwK4e!nI9fZrq&BVhpImN)NOx$2 z33PO23(G>*A@Mi=X|_Oo_$?1ph++&#PDZ9Y%*8@g<4-V7<&+-yCgDb*oo&|j&VlquE-dkcAsiT{gJnO-_RrO4R1=f=p$66xEtWlnc-Swrt8 z51?nJ7`wL?o*d*YFje`EU3`eN`i`Ca4t@sM?3dpIOpe`LKfC_`kLkR7i_~w1omh|q zg~MF72yrx~G=aM1;`gPG(>cDX7?UfNh%3eW%fumn25;LtPxKuw<^NZ9G&8aP5)Zj4 zY>%0`8dwWjXnG-A2WBtI#Hjs%pe|SjtL8z~vsTe0-!)08M4jgMl%hOF6?ia=*0D||1$me ziip5@{wGY)YNqYmSoCvrd~biB6c=FcQS4E7kK%ucW8aPLf|pEpzDB%y5$cO$99nQ;OD5`9YPx1>x~y65?)4R$+v} z*TGs%amX=He}MABs13rcy&|kD>IvqCYcOEWQK(sj`RNhdmDY7*f#^@#{q>@I6BJPg zVfsW(&jL=Ai;%LB4PGYwW3VY{3NNil+ncZprVG^}mw=0kEqsBf(GLW^e6$cwATm-_ zwiCCEEASLjM#_PUf>Rf4_jn{(DTMrB?X#Vr9Y(X-yv?>LlcqO1UatYpcyZ&s^`*+x zZgFc$S8q{YwDIMM8l@3GG6<`AN8A# zK3&IQsj@{~;W1zYqC&Bwx~Youvivs>B`!6;13cQw`3%-#DApxHb`soc1JS15=OUAFRWZ*+L#nscPg6voZeC}y)e(w@MwEfa zBNV-3^;eT3!@ssxpcf&?81s~OPYqZaXNo0l-1v>!WC2YbZhT-LXQTz%5SSa9&~eEt zDKODZ96M`)P?53;`Bh&gLe)Y~Hlj!Y_U;hCd22s9UkVEmew5qu1e4_lfW&)k>v`s~ z;DEyE){0YB)q8nPt)hR(r$^L-B# zc>XSJY)tHl0c;E@A_LHWcGMuBvmw4vG$0d_oYQl8`=++N1wv&x*vAbntQqub68!Pz z(Ri(S8j8TSJ}q^U01LQTQNuuD>;ifBAY#h7^Po+i5XiT~4rja&gc|SOQqX-xEH;Zk zOusfu+luOvzPkCaU*(aB1Nn;KCHaBp2y-^a6HuhOfyw)D;FY}C#3!5?g@0g8&jOb{ zoUhN%mm(hsF;CicvO0S?*uD1^=x3s7X5igQ9vj0=1pv7YLr(c8%eO!(9{@UfSy`FJ z@7~Es#Vf$(km5gp~L62cq9ZMEm8_$6PV_Z(32?*+?CO2&K9} zozbE5WuH-=XD;$%&NKHc`o5D8HCsxfL|v#mi_j+{m@8KXSrrso+=;|8+aCU$6ZRnx zSy_OO$NNhv{obz#=L2-;!xzB!{Sn7YOexHSZg773n>rB35M9B%q_tn5B4!$FP24>y zo;+CksW^4U*RMSHmrb)Jial znwpZ4lEd-8!bRNt=M(g+~s1D^IScsk?~JTZxbhV z|NNHdd6EG6@*s<}Ic7mtyy5YATpSp8o0ABc&kq$@1J7}H-NKF;I1}PVM9#UM8isy2 zD|;uYtK0gW33m_;!l^sEpYEwlP?YKC2T{ojjncH?Xl601dAM$LPM8Wgqb#-cCgZ2F zoQGZ*O(}YZ=4a0WsUWHLSG!^VoVa?NI7~bz;bBAm+w4+fEeMY>Kg_O=f^r=_e=v#^ zBOt%_TPf5r4wkb1=w+@;vsefHg_|Js-Y|c75Tgf?Mz^Hu7Ru1gjAZ?!=F~ntg^Y;& zTCmJ1j_3z;Q5W4BvrvjF0(yJzXu{1uak0KSv8$ILWkIJjohyoeu#0~Hy|=OwY@xI< znY3KmVzH=!BNvKGd zG58lTT~uxGTp!s_RXLj44sn$~(09R)2<>+@A~Z#m$c4Ju3dyO7=| zu`U)~HMOX|&j$pvBhOW97Q79Zaz*xEe)idGYhP>pH`=~Ewk)`DcF~m#u^GQw1(YV>vd&@RBD8w@cIkvTC@K_$$)|G)~IYq3P8)htX*@Ef?tBfF}}75^1W zhYcZyvSYOWY&8j7B_z5;mdc?WIYl#6{inXQ5mv8(uK?OG@?Au`h#B5%61Db+y_~rx z5Jx6zdE+wpU=f1rK!HOyUyO+U_=TV7H>m`*aA*lqckmd|HLZrxMMqhgiXR(lI7ZA@ zK<9FU+dtlZsLnL$AU1(QZ9OA#0N7!lXxyFil}2cGM!h+He9l^9%FT*`4-CW47GHT# z_5~K0j%xJw{HLxFY}Apb0cj9T9rsjTQ_3q_0H$MKuVw!3jDBI2zkLo>!%%P)W1%fE zP|=m$YLTY73q%uYR$5tWGHf2^BA=?tb^wn80MpRAw4mHr+?3IQ%(_gn614@h22K_1 zoI(d12CIPWEs^krMsE9;;&WW2!w>rI>^JL189O3X&7_e{*f{h@>W#8QAq<(js`r8N>1@?%^B7vxpY7a2op;MK2C+m`?TkPt&p;V47*fA*+}a&X>}yK)-;zq9MJ#PE*_~N94&>Tu$d`og zOO3Wvnz)c|>5pkke@O=_)*ak5BGJIfeE-QG#M^UV7iqK+xfYf>T@Xi(4TMrKi53j) z$I7Owm&&;=W01=q#a?EyOzqg|eyAQ(wY{KwW`S|h^k<$G$ z38_c9B^v1)Na+|Paf8>xw7bk>iWDz-wF$g_chnC7&31eZ{{`}BfTuvW7*o5$7Fd(q zGJ|p@n0WA^+)U-y0Ivf)PUT1>E9zu6#VIu<5AA7L8112bAhl>m&Vi6D40{XAd8kuz zFUR*S9s(4@Q2^BW5@ufoMyH3mzlp(M4Oafgm(1N@*p6AkK?TAL#v_+q7Hs}}Sb2Qq zH#MmStaLMX0+`vjuUP`X6p=1Pt1vH5G%z}^f0n)W2egIGf8`rGntKid_m)YcB1cm) zU4dd_YDrZy-0xq9W*y2i^3>gVxUbU)CnN(+-ri8()la(=*uch*KxC+H61qYWDp?_{ zE|(3b>#pDwC18Sw?1W_dvpdWD=hj}aFfoFEdxr3fH%`?tV6wB)BcrgxJIGf!<*eNEIY z6(IGtFkCo2XcQK^D@eQY*v%e=UU|ge8cu;GTwQgapZ4g}D5_h(CM}CQ19$dlh;VV;lk4d#)Z5s3F7 z&SzP-qngGZ-hHTn)8~%n??(^~L%|U6Y~=B4d&ietH{-u8L)FnyG~UONG}+JmStQo* z$@s}5=a+Emt#FFp6iN*;`!+9zuiHpHT|t2lJ?`$j!lo2o{;x*x?}L+xJ?=g+5WCq? z5VvdobOb~r>H!}+Na?LVRcf0C1tMQkE?!T`C5VP51YnoW9C+zRR<{!nLE8MJxLdLI zJB9%^5E3bDNHhW1s4(k9NcTCgPFK}g!3^|nbz|Y ziNqomK`3VYJ2yIE?~cewC=Rcn%Er+dzRq%^Cz!v|uBh`~)4i_zqRc@7*G9a0AN$d6 z?uZ&Gawtcfl2~fy^=oe(-=&(`E^)ujRR8y%eD|`=x7-{*vHD4|yeIezpFxeivJTts zc-u_@(~a}&OB^fzSwF>H*8bu0&#-JwUzUN`yk%>uuqYO;^+36_R95V6d^mm$|9~9T z&Xl}>(o$^ZIYsy;(tSujU$)8p1`-~V5GwSpwA%I<_0$$B#}@NucvT}M6HUyF4lH%5 zzeiZEpe9rqm!=cE4f0mlt303X58_o0LlfOm5QPA<0 zqI|=mZBV3>_%GSnbNR=Z{R(=${1X?RQ;^37sVL#)F+|qE6oyE_(i{eWDO!{u{3K~e zTSvV+7GCKWAH-8uQP?~=uvo>bJ2F2g6Kq4y-c-RynFi0J>N=>8xf91Ia(nS6l zY5y2J(sxpm6GdD%SQ`lH&|e7uU3Z>*{9Nhay6hW0eq?voVQZW(81)u9$l)u^63#2h zIhjDI->fM{gJ9d~Rr))P;VR}Mf{?%Hj$Us?RUOIt7t`_ZaJoEZ82QzHd5~(iPNAfu zCrw91)J1w(3ca>A+gkT6f>H6Qn@kp5Y5#Hs-Cz6Sfk|SmW`5!EZkEbW9nGSGSA{=R zWgJ13%+vAV%-4J1AIMI<6lZnHRh0%?YAg+myPCOF#MzrKw9k4{=48K(n{vOr8WVRi z);BGe5(!I?ERWN62ERq~A9?l~fDL|n4uJDK`t0ooC3*e>{F{g&MsaB+NPY34HM)L$ z%z)uETlJNf>7EQ^%LIG{4W$72#SU4ON)OiGMqfUU+l-YX{!4l@0|HCOag+9-9^?77;U+jJ>M`f8bTN$9h$0Uy3NYA9rN$#j*H% z1nw2C^B^KR1Xo!4(1An71nw8`lzLAv6jUKQSzI+rqj9j@{VL^gEttkAD4{MoX-qg_ zJSu}QKU#%MaR(N}2b_923HB&ZtTr~P+kc=G?jfiGxIvsc&a3iuHk#R!Cw>2mo*aZ$ zI_t(#BDOmdidpbB^1Ph0oE%x2-h24+G;tDLp}z#^@{iSC-t>~G$XgJMokS1((1mA= z-0CWGAC*Q*^hgOQI!*=xc7~4^3v$D;;E%W#f*f9|5|TXWxCTZmgCAm`)mV@mm#b}H zxcwK@lLZ8SaAo8CYZ9U0#~T7_{?3-yJ4b$yBW~wG?p>ScbGH_9v>dKz*_H^(vf5zb zI!fKc@$urPDp(9uwX>MsImW=A%D{akjH=6a(AgVfd}10|;wp%O0r5y25Hq5Dlol$m zQK2VR$ovqilFhFEY7rEZ6Iv)iW{zzdi3u1h;Up3*QY3bkof)5qt&jJ%g|lBrxJxvb zT6reK(RAQ^Qi|=o@?3I>iE0M`2G2jy(GKoMoqm_y%_+Vt+(bt#i2I|DTAuzZ+Nkx7 zIRYJQsFeO?tn!_@xR^MZDD|EC(D)#k4KPNXU7Y%)@5j5ir2q3V`P|UE^_v{*V(N>S zdgi#lvxBZ5dp2@#6|Gs%^h;ml?JcL3&pDFw6`vL>MN4&`9hFGjyRnR?VoHR#KW=h4eGD4q`DTIEEDqqdzp6-F5}`{(7NO>0%*JnwH*nsiwo@`X5v0Of@ILTN zwsFO^7h0wtv`PoQA!-@+U_B(d8>1kAmU%oyor zV#WT}bR$uBTOQS8Zpf?S$;Surz5CnzQRMl%No&+Qr0UA)k2C3ptMw3A=`S+6TxzYd zt4C4ke_RkO(nD$gkYIWIhkmVABSLBPwm(0rp2}$ICg8q>%T!I%H=*n&`MD zQjDPthvW8!YBaOnDW$b-Fw4-o41OsZYDqPoinnF{4`c3@q$>qWqT7F4rwUaBrqztn zo`cyTpmgKo51}DSrb6*R(3`s(VsSdO`Sz*w`7a$(RegpggL%!dg6;f|k4nQ~%yTPx zbdswnIRqz+qte(kTo~lEZa>Xf&UM}Lyq2-&txRS=WDXM2Z~Ixs;ovD58q~~ z4#v&|5t)T=kZF0LG`znb4);H~J-;9BAFFC-eSdIV`)_9E(Rac!UZ{@Yz`O)p*S9ib zllD{QR7krY(c8&v-vH*zCRY0=d;wl;9W>|r-Ta`EOT)2<@ew*+5GQ_kXtMfZ#mHOK z*=k?X1JINku@Q`#KkM8H>Y>9E83nW`2r?wNCi{XY$Qt;9?F7XfcxrdgMumEQdgY(Y zQ|bvO>GS<0JmWNA4pLssrLh7^>upgZ>3KOU6Ocp2=ibo|M9cm}I18 z55Qfn(HaEm6bH8ADVFcs+|in-*q4s)9CA@D5q-jxn^_?n2TH{1e3v21Pia5t7iscH zIeZV~k9z3l{V26Mbuv6e1F(#C)u#FfMRVE}=N*t&{r-PxhN{I&Tbbqf-Nt=o~nL*H+n5=!>CHR3cHHW6LQ$>bv!=;IR&Xz)RIb%)us}ZE1+A zK&Wt8Ctum&(jB%M5xWva3>e6{MX<{NCZ1E9 zDl5r|U!LnlFhr07~h+938ffgM~7zp-}maIdyJ`^J64N8&Tpe$bpnQ2IZA7%MICK(fXmQd&B2cJH~UQ( z=fRxhbzipq;q_jVb>EZHz(uR?+Q`XBN9!MoZu;bRBl2pd&j$Zi@TKnjs}XaKj`l2r zL6XaiaJ@|&ACEcMRS%~*NsaY*U;~-;5GFh6E-Gv{#V2UMN>dJPI!*nqrWY4F;Rm;G zRE(Mk#i>{7#5f)@44z>8gb@=5B!Y`~jM~i`Otru~5VxYS zgjCYN7G4>0Yp$|!F7sH?p=hviUiK#;l|CsXj9k5~{Dgps~OXE<&Z6}=ZUO~6)IM{ zJmS}OzPVUVmRaC%=e_F%$s2GtdIfrePZ|WfLf12=eu3+bs>qCqW!@*&d7! zMh$q?I+|P{r8n2#QgS906A{Ek{>)R3 z|56nRl6VIefb9&c&$T*r7kpi9DRLBPHfWeH0%pMj+=B2kf#hp2P3l&-Jl~sMGrN=5 zXO7Wfe8{U>xrzlvEkq?X_c($XLJSbU83H1NU7Dh5S&!7GJ#~6NdE?aMR%5+t`^1yI zvos^5gNuJOHq~BXK*l6KhJ|;Uy2Y#C*s&DN@W)mH@G`jIrE?W%Dp&MwK~v@CB)13M zWu#CQMCVQfVD!58(6>p(QPyBpdK+MQ$skH`IXY{0to*ja5Y48A_G^>kQ(E9E&a7W) zzz8=%E5p#hZ1KNtI1};Eq0&K7feua2!Lw%rXnj_+l6=;KIMvt~kRM2cCvthT7Ag{e_d|9Nsk z)h|y$0e2Qc30CFZ2&kG4{{3qiz~+xrpoc>ZbCc@+)___{6nPt{HUgUO8g{%#;(Q}{ z<_?B?H)S4#02KfUk9a@se8KU`+{GxmArbG4`bOm+y)i+S38?S0#z7&}pASaR(C6!? zwwu>IW)Vq`nu~L6i)(z0wp?~LdtJx+MAMRFBW6ihdYU6`>z49Zn{b5{@^;oHWLb5# zBJu}Uotf@Pp-oyD&m0LzqWni@l(k88Z*+57MeFn=ovQI@F<&WCi_t9^wsUA5Q~A>~ zz&S-Na)&ii$GLjUp`f(@f!2ea++&&MGWb;4eE9^en?@GERKWfhq!*^T)xg`VS~XIK z)}#4!xxh8$RQvBmHLZK|ilUT*SRa%DuKMq^ zK@r4zKQaF&AElECTW4pN-x$m=!&bGMv;yd}%G=+xAtk)i<^uTI5#|jn}bw>CDP2awGgh`4r7@+y*7A#P#@If$1XW5|nKcXE{s3Y?nU_t-z zIgwI+1ir*Spz#hk6e3+)|AHFh>@87x6+_KIOd}5AO%={N{ zbC%OUw20+mC}eXzlW5djntRQ2rNc&7UUVmBB4<`Jkcrcr84sp}sk7}FfD<;J!r$k( zdF8|()*%=Zkw@0^Ud<1COG3=43n@OxuTz zFdwy-JGr+%q#lUC$Vs1(Z!SKbx0?^;m~B-iY8s~!#d+?3E|%;bO96;wrYm%@IWDQL4jxp-l>CCfG4XilkyQj7$1n- zo_n7-Rg9`k)w~`&`j0ue_?nfi>YTiBS2ZV<3oc%DWiaEhD8Ccr(6o5JksN7qP@$V% zbbjm=hvVa_FLk+j`(>ja-lTCgeYQ%$f=cOAtN``L^8DA$*YzZy&f}8r)lQ^J=Qc#` zgl*th)oY39!r}OwJ@ZH7iRP~-rO#__#`JL%oEtVAWK_StpZHJyZ$6CetS;gm1-2-^ zTh<4wlvQ#MKM0RUn89JjKwHKQ3^aOL+X8Gnnz=GjYVlxvG1;j$3>7N7IG%|4IHdu1 z)N|=6z3y5I1O?v41U3v$m4=10ltKOv_4A}#=Xd>DMgqgA`NTKrpo~>j6Drdo2dBit zLZdW}0kMw1PpvLYxA^#?($@4`o`9<_ziTt^CHQCWeMzHPtZ2K%Z=Sy8@7=`j;bHd` zfTvISfiK&A(yK4*k6w+tufUKG;%mWY_v`V4{Nxq%M~vdakW;D`Z1!@K)A4i9pEDqp z7R&Kzd%;{nUdaD(OAbF`H%m$e%PSE|W zdB1KdML>9*i!C;X5Z#(lMWOM)*}|9zdn569pyjB4(9S^7Wd%|#fJ{9+jYlz&1XFhY zyCPlsx&Ia^IeC1rDq}HY#$0fx_*z8T3&Cwx03l!TK=rVzCv+`D2p4;hN~|g)eMrdc zeAemc?A-TOj6_&{I|KN&rJ#X*E{o8T)%hzTE0?lD+QHell?`vDn(7RRrh}YybvhQr z_IydFdnOQuxvc=xQwY*I1LM(Jav%AAv*(eeAL#H63cs2?(D6g8%*rewFgj_lEFqX!FXkURr`kP6@krgLQ21p+n6Simde zWS~l=k?4ws;$M$D=AHJY%5QRcmTWD0s~!Jrec#Jf9s*2pe7r4Jz2^3hAFU4)q5^&_ zNlSim#KUq_Kp}~*)8XtfiLVQ5yev+k^VlqK(C1iv?nl#!r$K&hUG!4;ye4BgpZ?~^ z^=l&g2b`4h89)2l-~)p(h+aE0V{r3~D;;?b4F8swft_l8{FOAG=pq>ig6$u@MvV>*r*9<(0U}xe{!V&L@Fh$xL0jxDZAy_mckSpZr@M z^H*Ow%hBgFoP*09jZFI5HmIQ&2~|mxU>u19qlqP>1TRZEJMc6V@ zB~gn-p{FgOOXy1*BCtU-wx=7`hmP^uTi!j_*WZgROEoPS!}AeFR^NDig4?=DWPAbC z=^-s>nz7^;Z^DQw;;_pTA{4Op{s2!w!5`8suoK>iqtQ${SRxR`TmmhP%>bmYPYX3pa9 zTbcs?i+qdSpaMpL%AM1eAXZQkkn))(mM})VEA1-nWFFOqhqOoz2P}gI_T&8A6$DHg z6;;-T+@uNxmcLkA=0{Yi%3$x#n>tjjgRSvO(02P3M(`%PCTbgIO_GS}VX1IRLzTgfg;<)NiTf<9-RN>S>@g8>aS(;l!&bE4DN=q~iPIg!|*6OLzOdBIZza zJy2RL;o`B;1ra!bfRsV2figGq{C7{^lmUhYVGg82OyH^XUC%?3#^q8ab0fF0lBMzT zh$j>VKZ<(#CaRx^vD(<@0#vr<{0KDGfkXiIKjZ*@N`w^P+-DbdEHUEN8a4@v0x5)b z8*al4F;NVUPCS|2^J%!K?+0cvbtAl6Yx`^ny@0X0ZhL6F2}gTKTtXxJyyG8n$kNFt zUOV^cMd~IwMabCcX-v{%mx^q9zCa|F9mLq}B^kxWIlxd?@@Rk5@s1smBPd2D0LB5a zOUk1r+}9em1d~Vd^OK8o9d{+5b300=FMxiQQjdF2S^gMZw3USJd7_*jri zqygJzXCz>5Q65x|9H}Bh@*9u2Pd=wJ*boRz*A$gq0DeZ+E(>Qx3u495;_2wcQ!=p( zn-ebkcMm_~kU0e-+mVCD>Q>lf1IHp61*smbjIztn6!`IF%oRc0CG2BxIrN4 z(!Z^JD*i1CFTZwF1HYVlP*JrXYgB_-@xFsW7k+>|h}6Tcr^uB>)&24L1VOl~xCrs+ zA)#m;*Ldn<{#2A&0%KDdtL1fG>k4iljzEk%ByfsCSdj6Fssqwc0z%3RV90n%-TCm) z!FrOhB0r_!2nD%epy)TGP7J0(SlKgHFeXS5QxpmFS2t>70!Qp#37h`P{isVid&&kH zthar1Q&*Pc0PJ=N7Z|YZR2(ssO3@CXLT1z+cevvX0NN(?6-?XhX+Rg!^-oZMiyE~3 z>d=m6{0B^eZyz4x+D8&Te_ns5AZ)*n)tEo*&F)-{mcG*7=DDrH3XI2$-jC``Jm-a~ z0+d>T_@MbY8AJG_*a$lN@g>?~Sh6k>;YzN*29wOm5d;V995nr_pE4mj4K0Hx{q+di z0qww%NB&J7L~;!%tV)iX%Bey^s#QVI*iLzHA)A@q3|jvNf0UTByAc$S!mgIffP$(4 zVLo!%TA83O9Sgw$|9i3MbbS<%Ig~_8S7l^t(~#CXQ9KnAodRCT$BC?hcaqb z#*Ndx^v}RZkeF+&Nsz4jr!#%C`^8ZYVA7EWo&5_eF%9hTg8Mop!4D#F$JjUZZ4E0$+wu=b1C0(! zA=nQmSN@~=g|n&h$nnthD~`sP$GD3K>t0>zG+YcBYXCDE3KLvB6Q%sevVjpT4FWdAy3FnLh!DWCAsO&Fs zye5N-(_5Na-3gIJCQper?3H>us2Mmu0Xa5h8!Y;g+Uk8!F}DzC*sU=GD-oK3H|Qdz zyRz1rv|H@rHbJ0ZGhHrwjMaIc<2m89pxUS}Buktn$&h1h zO|~m4HWTh{F<0V%G>~yc1dW1dLOD-jJW(j*!zzZQf4WBj;8!gS6&Im#d%YbKVsS}z_J@(J>BlO6f(0@*W=iAKJ& zPZG0>#`9dm@&4rpIJ}bm-n)Hi2IS}GBlP0r=;`MD3>Ad{ht=N`8xwCQ8U=lkz<1c; zkJaCcwq}tw7537x(LU#~t+->NPw6HjX>@b6@Uwd(|Ev)V5!a}tmp=05GQh^d3 z7nUHG#ehZ?fQt{eP6Q*7a=6{r<%RyC;b|Vu9Ci#LXZt~dY@KJJN^<}Di&kg?Y5c1hu2uAQDdr(oNJmQ{ z1n}Rmw0c3V$I#|G_L{JGMr(h3VFT3@Qit+TwQQHiN6a&eBFhDpa{0xF%YvBoMky*BH+ zp)`*Q9FN_b*osrqw~f$P^{?c$y#+w9{2=o8Lb+Xd^%+_b&cr{Qt$hP~Xqf4kWuNuX#=riSzrxJ)?gSp7;qmbS(~K+&m{W1r zkC{Y`HhCvTp`VrKmX{ZJmtC{QSyGx8{+F?VkO-C+#&p_0(#EC4pg9vR0tJr8L6hq6 zC0n1qi!d{E7fnIGe^jMI@Aa{A)lJPUd@2RiCiwAJ&cBZppaUG2y%+38+@V{B;?Xig zv^8MfVk&}*PO4Qg>+%!!EVC|hNzBo`+$TVkA@kR!hp&CSsy+iRhBwZOK3gUl zJ%A&+8p#>SQ;dp{)6KnpEyUAH5;8eYkgwX9cSIFwRCV7*V7D9h7Gs{-F)IH3)iJ_) z+L1ks??41fIC0_kzg@qY++G@nALtBp69(-Et&?9-MN-Ja;Lo04Jw-M}hA!rbe)B4; zt16GF2gKbUeDLGm9GE3B8a8C2UHQ`>HHeimmUy( zBzEk)lZJT0Es8(QYD3Iy`)6%n8#eJzN6|tx*t}WKw0X4S@ze|B4uIM2QOyZb^#!o_ zN*=Hp1^EBO9KQg#p3nspe>$uEM{l_=}@ zAy#GAxjL?MJNw&B4v_(`j-4dS(_;$-A;L zqO2hJy0C0D=_GM4(km#;Wgt`j$=?5$2RVGrqpAb;qXhaEnjZHf?k2O+kEkaI&mHyH zcv1`ge3Li!iCaFg%7HyXeXtP_kv$G zT*u{&VbUhcIS54Q6u!s~9x{qv;!OF!AdFAPG}+T#o}6AYUG@rN!hH!psvBN)O;M#W z%r^cN&M|>fCT=5Ah^Y6)yJx=HO}+tKFbd`m)+yh^YUFMyM>K-$a}Lfx4dLd&TAo6 zk*!x*D=38evRlF4_vH2J$Lt6VAT}sG^jXeN8kI z83>b)uNAcf;jh62{h{v7Y35snMazwv)~*WlVK2l4BW+o_2)rmNtk||eCzd;ZUJmIV z-a?csOkk*`vosTOx!`M@XAIeX+dZrjZbhi2`fXp$V+B6Os)jkbXYa7JL5|TxzBX;U zI~IbjyRHJp{2+?-hb{_>RkfHlc-fPPqD%Q1)DWg(@vi7VtPuhbP{6JSr>?36Pl+3v z3}ziN=|f z|LnR)D!v}fX5CM^Mh|my+1hqL6*YC(l59z`?DN{6B$@P5gen_j%VI{gsIk^_T|8Kd zt{uH)@A9->tTxp1EoOxn$_^4kgv2UL1@#rHKEzTqY^jJik5zv>kO7Z$Dk94dhM^xe zdqeR~(XR0Z{13WPDYIQ4lb!#P&DCeK>UX@_F@PZF2lbV{@sa;-;hPIE`LP()d=hr0 zJ&TlotiQPY?3Mt?{w-0Wg!gmd$rmCPuj?Ohgme8)k$iputGI`84zaLRFfjN*0*0x6 z0VCIdObdtjW_yoeX*~r(Q8lTTR=EOldk2;iws)EkGch->!y_bZ^CTsdza9=i^g_l} zj6#%4xMh)S@c{5|^kY;x?zCSE7S^i81iH=aan>HK16+kvJmj{vIA@ zN-U4qFG%c(IWYtjV@&Pltuos;Snnpoox*DoxR8nB%|Vf#@U?kq0Syd)q0Q%Ja*5cc1bZI_WdOpCa9ZT$9U_o<^A&9-S&Haf}~OF z$CR#G@4Q@T^wkG7nGO`ENZY+?KG#G>mqve&wIltYZM69gnQ6~vsYaAj``te{N{Hdz z*U{TYlmiCnNIlk|w8wv1c40ACuor&|IJ-3tyEAYgv%*wt*tRu^$ElHyc@N%0Wh+;G z)yMlh4!Doh0FJkwd8w4u;>A;1NG5HVF^yM?p@IA(v4Kqg@cbE&EeOs~VtTOke#%I^ z>bvrr-=;DzIOHPK6z`Qt3_Zap@0Zw1#fMx6=P-u5gLWCP+N(6tC2tdOUv8H!J4S@! z{10oF+@Fa@Za!|r^7}aM&6Jw88kFIEADruY>_xg;`mx;QqrKu6#bZI_n+tarqqVO% znQH2~wcB$sc8XPw09{VH(r(*$OT(|UfC2s_okOBGEiwCP_ke`ziw`=n>kOt< zD-^h5Q9QS;V4T#^9dbApiYd4&5lfw2-3>UomjbeIc!vh3(Lxhij_pogn)jQH?>W6c znzNYTQ^Ubi3GIK-^+Mt(4fzE;{``=_e`Rb8e?u_A6we6r`YuzZM^QU+NsI!8djDV% z4f`pf))O2Zc(nLJoXp|Xs(+^#*E*4+vHqR<$5WKDe_SXx-PLc`#?+PFV?d(~Phzt^ z=_ac{>B=Gd&xU$1vsvn&VOauREJJuW0}Il2ij%!i$i!NKk~yQ&Xbgqdk|b&yFXLti zIca+(G)p9gOr50`;q)=4>QP}?0vSAKI5_7mwEt-wGHUfb22UQJ0YPxkA&zAhFk9de zT=)`XMpoaG=w3)|16s0b=cBZ@ddI?uk3^lq;XhLTrzn(IX{JmfOKhPLh)3RI?Wv>4 z&d1A-SIq!TNcAH3iM+CpD&E-9{~Sq`vy(&q;vNw*OoiM+5Ki195%k+jWEYW?lM@NI zSWmX5F6P;3Mp0 zoDe^(D_#XdeWAyh4X&u(NM|5$B)92hl!iOYkczWTpTxkzxj^)$5O9xiwCeD;3iLd) zZsm;l3it{rODNhX?_Z?#Hb6-Oc6vaU;;Qk;E6p$g`L;|LbpJ_`%tAOB2(TG`|K9A2 z_bNiqjK@_~^+lEj1Xbv!flJW#ram}*5XL1>614b8kn@uhYq}`Npi#7NVj||cOqb!C zw&_A8hEE}|v74%9fPOIfHXJCaiY0|-4FzYQPEZnUIf`830-?Bs&W1`x;5ac^z~c84 zE)<*9-im=S$90-iyWA6$#-&O0a0|H&+iExD`I}0$v}1HDpk-LjDdyA0^HB_`X~BsF z@@n8goz|WT&b^C@ex;zO(I%|E=4KgbhfrL!BsxG+Gyxi&zi zIQXWvws7jQpD?rWXhG7=bI~b@CCxziC8JM_QQYsqat$^!^|s_NHgeV2S?UGCyb+n8 z87rzSSXeX|)ZK83~?L1U=H9)q=U zdcnlW?W>1d{pYMX{|Rfd_g*Rc=?Hg~I}ihPtON?LAUztDM8Y`B=SvX@E6NM<7YLNB zIqclT9f(;XS$MVVcy+6H7|gPXuD=48wfohQ>qRb+;FpQ%*n_?9)a{y zPXpd3a6MSuPr<;!(<)_Ur!Fz^@0bSuiVAw?LC}8kA01>QrQPDRj{{ zBMZrq%v=J@Ua$B#PvMToj?|vH0*ZG&Us< zJ%NxW*_0m$CY6cmtV4zo^cjyf{KDu1VAc0|I+V0S#8S-&p+T~$dwl(-jskO^B`15{ zG#XL*O`$Ep%|Q#EQJ{)o7;r+0{m(Et+&IcwIN^cZB+fE{PYV98xnYIJIXiBGwAG^9 zj1h5iXX_p2mEkw*UuUxh$<$@ZWjwMZ=EJ8#$q^}5CQ*t<>?EM$_gN$*>>A9^C<3*d z#82)6Ot_*N+W|_OaYtgC^i&0=`r%5l6RjjHN5y?VxOax#J_)O;L1k)fnnfeBqbc}! zq%Ejw)}-PlY&03hHS=PPIE+~B@KEh0_viris)Es)-~l@F{osmAmWm`f+RSa}ZzO#9 z8wn)=sKi3QB>FNx9lq{~G>#Cv0rLKzFwHQR-MC7I_Ug=ND;+h?CIalH-AUAYpyra! zC*dEcQS`NffjVwO%-{Sg!<|gONv?5aWdK_0n+cjK!3Tf>SK`cz5=-$vESx2!5&Nj) zCeot0f~uWJE{#~NEG7-u$29siGhq$NS!`+eBx#joTG6Cho!z6%u_h;67-Wu`v=NmY z)*_k|?Mgu!yYG_=4Rl%L62{4jY5KJvK&ISLY%%y4G1Tt=u9w+^vGU&9CQw4z18ULx zqwI!j9*~UPQ#0J-lRO#A>aYhe-EIqGYbEEy9iQNs_Wu2jsbSX*!bakDol%NVH* z%-#4y0V(-xK{mPgy&?s`1^O{kgL?j|7KVb`slL^Cl!bis5i~OAY+#Emf_iQYnbDGbpJ{KLWL!1YwRRO>ItNd{>)!lv)*TmB| zZa;z?!{om34vGIVR7P`Tv-i0IqRV1xcYBSqrhVY0ZRt_+2E%5k^OzhKIw)Wh2)#&ra^@07cv9G#l}F`^0$vYcRG+R~-wvsCV<^f6vkSe$aDsDuMK?k+$-I{U%DguK=Z%;5B*D2!m@)_V!I za9?j}Iw#EwU7Lh1!MKPSK9xN9zqK&5=l@>|%WKrxXWd_RmveBW5imYoC5*izwRF{i z;U6_o-R;gpHC9R+TY)xqn(N?@C}3hKe;FB z^j1>mXUmprbyG=m9SZ`Mk#XsBvaxpD25G;ld}`2f@c~_zf1|-{()|uFad~+_sVrxN ztY~R#N6e{GSrNFEGJznQo?bBTR-61sTBu}ag#%?Vq?Dg?FD0b%R__C}djHET*!oUX zUj0wl5uEo#SEmLT_BL~yu{Q2Ur(T7f=W=_g7rNMmU-A%5&r<9UxPJq#zP#PHxSFV< z0N-D4D*II2oQ{MtX~!DkALjK(Zi{>yOHv=3=b5hRZj87j#L%`;hM)=fay&21d!d-n zy~E*_6Vl`C$*qzFia;=Gp~7lLsPyE}Nj(Jc|ERvWRL&kY z$IAyW%V)FQ)=H(Gfcwi7{gNwc-T74VEG^9B&O+_%hMwEDAC z7=wewV_z&)WT!n5^U1VCdY1NFs7s>klFs|<1O$L3qpA9JM~iGuq1P+%FP};n6C|g4 zzXj$dxon;cI<+8#?d|8^A;{QYy8_A$o=@I{G7IqU=8b2B%1n+1GX$1{=UJF}3>&D;PoOHFqpgIt{ z)9L(u?gXy4YMS6Wp*(q-M&=$$W{{REKVLhXz+wx5@JQr4f$?v#HjN823-qAs2CV|7?aP|59m#+R ziH;#@q>v1;LYgq0=kt1btfTScINGoIE=T6Bu0p8JzrC{{%6QZMdkWijy&w(UO*J5< zus0XBbs>3N9GzB?J><`+AKPQvAn-_Lb59jyL|iV} zu_vYenn*LvXPV%D##Yj8zC&({g(yXf{QadR*u)Omn$eaugj(KWn|*{!G|HlFE0aZU zrG6Lz>~e!-!HLEVwMVKkZVai+nYTk1&r+Ma{j5j{%TEE@M{*I0mnT%`yxvk>p(o~w z^{H(`Yfk1#cF0Jpv{`z0oIaQyv5YtAEcOz+u0R`}mCS5bYO>#lo4Zkgm5zuUV0{d)>u`degAHV+*yq~P$Iu0>FSkPU?_K$i&jTO zx!0+NVPL2TZzSYP{+}Dm2tKe4Hr(PB{skSmm~2M>T5r(Y)egY2_6+gwZoc0Y%~i1M zHh(_!g|lc-bIwo6>hju|aL|M(Kbei$QhR+=H&WG_-Gf7O}{rCN6C z=4q(a3s>;O-)!Ds7}C=V10H!vD9(lyzHJcbF{;o@?&_jw{!vE@x>^N&heyD&jsj6m z2d=ApCaYPP^XvEuuLrjWLMNynwJLgAX`N3s$}$yq8D{ada&VnrThFBzwIuoOO}=AR zM36~gFA8v-yOuEj{dJX~KFAF&8Zmn_6JS{5LduPP&yuf5G=Y?3XmY??tft39>;YXE zTC8iL?3z?ZZt2$Vz>xBH4xtM#B0r*(8SP@A%Q3vqm4>0z9Dg*cLB~+=OI1U`IFWqU zAv;#rK2#t^E=fnZ(%=L-e-#{kOmF?zt> zaC2xy7`hWhevkCU$Lbrc$nJ2>@cPO2s^WQQMKQBiQPlUZ*?yddYlXJ%#UX({4=n)z z9F$tm7px%K>nG+>vAe@913JgSTo85iS~VNU8LPx{_->aBAINHS5~*RZrU?Tt$PQs4 zq%S2-z-w2LlB7sQY<~+|oo-|gw@k6IKRT^y|LTVaWS8n)`{p&!Jo=7mz;X8PRH2tK ziR!GwDWJgF(84%*pC|!ku1v$zB#(7140t>9QZRWSu#g8$Ss9M`5E^o8-vDP*alse( zFRkr1#9^`YkcYU+%S%kLFXodgj@ljvMG#R#wNqubR4Xo;%8RVTB1t5Oq8&)KRtaM4 zQb>p%BLPO~xdPDO{Uj85yu(;iMYA1#oSN^@(RA(|IikG2`IOPm*?i?;<0NjJ5Nh}w z3Cch&0vjY@yo6C2`wZ*A3W^zF*hC;j{WyxtP+6Z0{k`bM@Qo|b0}ZIoDd#nG4^s7Z zRi_K;3OLVkhy)h zS}S-s(V28v`e7NGho{NV&&&o{h0#59S>bS4Rwz0>XGugnm;~bdh@~=zU7IM?nXWEU zCRQgNapOS-wB8P4+fVvi{Z}J_rBoVsQOIX#cjg0%y?)m|m)`&6c+p<~gyf{%qYO?& z^fqu*>i|OhT#6UK*iiA3+s{j^y!A^Xa@eNW-Od5DUi+pqgZr3Z;pDmPo;rrWp_cmv)}A(+8?-nbu(sAuXfdx7+wE)3 zi~i()#$|e-@d(QB0N~@}mOBGT1Sbns|CBUU*8Wf(xz@l6okuawzB*m_o&?(N5s?ix zI;q>|OOWmlC%=BV>d^G^k;YHG#-u8kCL07E9pHvj>={MNs|Sl-JA z0G+_Hb!A7rNkFd|=HAn*Bi&y*e-aYTLIb)jW%CrRJaX~H5Fxh4yG#+I0Kdfh?p$M+ zn$<9^tpx7l*{gI-!@YN5OrR)Uu*|Ic(;nKVYRb0wwB$c=4>8<&?p%5WadI4MRw3X8 zNPBj??mmu(9@)dsN3TSC@KDdOQpPI;Z8^^7%sqzrq+W=8DhrX^sx;IBMvPa#4_zmKr-0L3 zDnoqAbQ`a_U=dg`eS@&5wH#ce5-aYXE(?1*3yU!lJ{?2Js)xQ)Vg*|YC4n36PXi(K z)AFj=sB9*mV}ZsqTHxRp0&~0*Ap_Ctw!mqi1I(TGmuBEq^HMoE`@)raC#%S&Vq&%i z;oXLA(I#w^|0++km30)nT~xp_EP~wx&8!UZPmEpAVP%yeS}*CAlFKzyla2Lao4)C^ z;dyd$HU0H+%xmGdpWOGtHPuk?oe`zfPlN`5b;Yl<1dTRe2s-JGnVuJ?b>CnNjRnv^7Qa9hGyIyMX z^f3{8$H)70)wePz81)H|kBd+G_AjR#TK(>dv}pO_{54)&QE}8P!T^l7<1Qzg= zpjB&@?wV@5YMBG=#kb^$o?s?RZH3+=pRcS)^3Oa_suZt1Vy4x6zuX+6PLc-vqa4QW zdLxbAyYXg!wIc7^z~jXKih3RpPFZdt1&A+hUIo~bD%b+zC6i+q5z{U~Ut!IC5S`g9 zXeQL_M%>9Q2V$hJDTbxzwv{ZI(QaslQQt~FFxIq8itl)X5~p=^dhQ<`o;4 z{#4jbMsR>(#GOZNw2waUG}B4`W(Nu^C3mlVTzuZb8rHlQ{v%5Y>?mN~n*54nL!(vc zV$(sfD+|A9?HEdnF#hfC!Xh~dSmFRo zndiBF2{0n}EP@$jet<2XF4prM~q**;YgMVH42u9 zKM#j+-IT44Y_5)^ggxRB2AoPh`W-l5Wd_d7hc^AmTe*!yD>MxtEp*LYUC9BACSLT=?mPigALsC$Mi`a^# zskYuRCP+s9romPV@-A!EPZQ~5IYj}*G2?F7ZRB?4IqYl znRQqOeW6To6`>Y8W3&2gsMuG|Yw$m?&)l%%hjzg@nN!)R9#lG(OkNUpI@7U!x)IKM zh>zn>nL_`qHiQqo^v`Jc`AWjaZ$0%TmcPGqvUWibLPeVJ`Q3@G{yCpX*4Z-4qgd^W z28pLT9Y0;oPxEzndMA8Vc!&PjTX0~yYP#dyLnDG#v<~RRL>7K_qs9bluF5Nbc%$g9jMAHZ@_zf z=a#G*Vw^U%q}$;BRz1;%n>lFug|=g&^l+QuD4hN9 zGv7x-SkEi4l-ewAJPC?``a8~pBN@mYQ6jfQYv6=;Tu#JOZ|XzSp3>EY}{ zLUiK^{^2V!bNmc3T=RMP9BVAhY@qLv(Ab{H^<+iAo47S^O}T<@m4cRbZYo_GDe$XY zxv)*l4GqfzjvGdOy?e%x~9w6#wy@neXML!-VV5cTI@X4?l zI|ldR#Kty2?&XCv-_DT$_ePMH-D`ykbmpM4)Wj2_oGwRyE85=Jzr*1C= zr&|7Dp3GgU@dp6--AjyFT$^$6S9wsUld|1dnp{*k;PXzspx1^0rohPL^2?gHMmy4SUB zQZE-&9;1v<)ctFKz{{8Fn>VIwB7IP1*&ET9OU&&vTX3Q9a|&|^UX=M>20;vg!`<@D za(uhcEF|W;5e5^HHSV*{?cKizK>>MYs-4I;*~qVW+hm9Px$7lru0|xjNa?2WWqrurvmfR}0^8K!$>7O3o9b}YhCwo_Ali3$ot)exrHA9l z>@XLZ>O1C@DzGn#k|&4w?0%y!nJ_zpB>3DwP##?`C@v>4?%ke}&z3fLe%1Q*mQX-n z-+kF%1>S_v8Lo$OJa}s5g=ICX5$;Y34n&mMk+AR@Yu~iGw63UT<@dY({P7^*uA(+d{6qSb}pF8JZOqKAzf zv@gRt(+l2BcT1hQi`yYQ(WpK64_pdVU%^>NED zO?7Cm7eIoKXJA%l_5azKOJd2{kUcPZYHHB;_^aKq4!%MT6GHVwk4}4G>zLHd#dOGc zX97Z~j{t){&+claj_=Qr+33}lJ+*=-v4&tKm07{3k^ZtGiK;M883t&wNU^@T)6RXn#$UecFyC6rPKOPO{lH|ek!IB~Jb`wf+ z7jZJ<&tsD!K;qv$c;13QIK=S%r*r>OW^K|;cdqg=y3OrjY(t< zO{sT%aF!aSK3^fS!mzPCN1}mkso%lojcou%>Y*8%0Qi)*{(Lxjxp*xfKkZ2h)&Pq- zNdG@eg;DN8^=zRI+oHn0B&DHMoxdNb99i$^ep$6`gxx&-}-R{(QS{U-;L4uU+`VpZhmi;3@*3juO8q{d`nl z3X0;(iy*SNztsQqR|wRitkVS6qO>QLE3uy_ek3R!7B0RKehfAHJ$za?xUJ^>EE3O6 z8qLF+fVjdXmC+!H#aUBWhXIR_Lw_0Uy)JKZ=l(DYwN3crC=adgC>yaZvoj5Ro%r)( zoOJ|s8~t3Lm1-6B7i(#IPu18^2Hp~TnK0n*%p@_x$ao~kT;{1f!)q3#(Z(#Em-!)v zlk=9?XXRve z7KGhoY#jlQ;MKse>I{tm;P$oPB+c28Qp)ZfnQ%14YwGe;xEua-qmq~xUok@uWu<5( z2Diwi>td^E359QbMVO`pf>DB}0s{q{T<1?+U-B3fo0Sw zqy`Ap)Y#mOqQgW39TZ8*!lRPQ5oK!f$K{x?VUIQ{Tb+2_TdGvQgEwQz8^_k&8wG3! z4^jQ2j{W^JVgNii1o>C(pMbI}f!fcx6o5b#`cf6Jf;>DCL8mZkV5!jy?O(S!-As#W zH*{V>Al;ft8atP)Mo;k1c1dDB{g7f>G?Z+ zhF3x7TOclhP(~Xb{M|qC{-%8^MHLJOFO8Mc8{@f^{;Kv0B41A{9pOw%gp@dtjBSJ0@(bG@76(o$31gJ znyV&Wgp`Vkp6qlrPXZpPOi|4fC)(HBW*zvgdjW$SG~`PPaCNh33bk}^2bvPv5S@#HYG0c#l7+z?zkx7OY9Fj^;a*GvM&mrY)QUU)4F}~ z$&EQr>ZIo>PdkrIO6MCEA@^(KSgJg55GhPk0XiM$u@rTkhE1F_0I^X+*D$HB7DegOoazDMGXf3F3sZVyO4<|eFu zmXe3p@-Cn&vAhMq8}@a&ux+dIH(!GP)@okCZOOb~ren;HQk{ss0GA!ab`ip~i?EHN zz+6u7zR{LmuonCbI(9U<1&CZDmL=a3W2xlJlrn}FrVcN;uD3|^k|mWfY-y$WuMzxx znSV=J$xjZSgZ}@V6Hgsk0Tty;->1iHMgVSpo1?z`B!GSJ{hlz0BBA4O?;e%|AIo}v zS$DKM8|=LKPLg1%;GMiUnMe5yxa!U{I#Qb)=%Xn~f7gDrGzfjecRQTy`j(DX(VvU^ z_Dq9i6)J$rslb@UFMogpEkNfDuzU>Qbf$X(LkqZN$DA&<0k*}-kfQ;fdG`mS8&|}I zIR-4;y>fX1j_%G}#!dinyfbBZ9#JUWsu&^Zt>1~rwz>)AMu0@L(ANZsHQiCjUj4jx zJvf0`d18C>Bi^-xlJ?JHj}p@ybG8f}0UcfypLjktu^TAag0TcQmiuu~hY<4;2EX+5 zyoO!h{q1cVfM>b2Cqxxh~JYMt&oqP*z(#;s=f$-|Sbsp!|}DR=uKOqYZF<~^)M?QvFz$TK`=3bnp@Okf9+ z72(P~C1DRXK{!DOf`rp2GN8`8d8qDo6vMf%6If6B8OC)39}ajQH0v@F4IpgPVhY@U zk+cCyU>I;8Dh>o-r!MkT1qpLtFsx_6mu)EOemc=acRxK}cwTOsIC_s?L5^Cwv6S%k zd&(7+Z=rRxo0wb^b@Jdjc?`?o&R4W;#81-)r^sA<+}u1HpoW9037-^OB~YDx@4F9C zNZ+W>NIEQWf6c@4F9IhxpV_Jb3bl^=caaXNtdU#qf^8Uncc0BDRZ8fgDNQ(k>R z4B3B2U5jK{LZlE(Fvw3kOK!-YLzDJg4c&z&bXmp&jPn%)PyL&Q023tJJo{$A)s139 zenwU{aL)q<`*Yw2g^1CDe{7h1lQj9GZkLOru$xm>{Ny1;!UN6*MUsGv^@l0E5DFt2 z@sdDp0+z_?MFbetvr4SbayLs_s^4kBP?>Z}FGVLtcfK!`VVs zfOGSSo_UBogH={a(Z*cBm2JX3R>hg?Qq4^&#LXC|e$go@!|7~dD&}@=wxyC?&%aBu zrzqu9w*jfXQ|!E{H3Y3J@tSw_w?gECA=m5$|A(+L1q}Z zxX`Zs-ZvY77Vl)PA80(o#XN1a!_Y1=g&a}D7b0%;=oaZK+>Jzof#8o>>eHScJ1QZ~ zLmYT&bUSdyp23I(m>HveD&}Q-DwW0PKLlpR5mlO&6+65XS{o6b6tA?n*g8AQ%^~7K2+++*bc|gBlQTQLf>PzEAHkYF&)cc<-L!C-MUyq?j5PBa!WQx$5zZIaDL%NKZ1)gTl0a=c0Lt2`#e_d zBG1e|rhcJ~1B#0Wq!z#3*?u4OaXu{n158GY z)#XW^2rDVe^VhZvznesv^(3M0V!!EQGiyXM1n8RzTAXu+qh5(10)Bmv9wjg?5~|M1 zKk;9{O7}?$$NM%1NZcNvz43mS@iTqxfQ(zrQ!Xw9mKl%%kMwE@X8cS*u4b)q!w9@$ zAsmnTsIkk#2HV?0sJlA%M-UkolVO4bN>3H{LnvV5E2j;_1WlhCz@S#5x#46bWN{jK z`aupUs=P8B-@;)I7$HYBLyQHIl1=_Bhz8SaXl0zqes)0V{M&`$aHqcO+8@TcbCP=M;a& za(EnWKeIbLWUTS{WP{UT+7e^LPIhw8wAAv{4ZSzn9QU0*`3;RybSGYZ|3VdQ2*^sse^^JKLIWdo}-W7lUX}Y{Jqp$XJQZW0r@8C;^N6?u8 zgqIIB8#w_4xa|J<+aFu4FUbcjQDy@)V?sCmeQ%*KP#|}DR0yN9Bg8;bJ5hKTZ|I;j z$$WN0Oh5VXsnHPgFbN=2Sa`ih&GVhTr6WZykKwHw?>d~)ozpwUMsms&(LB}ze}h3h ztnpzS-@DU2OHLWw;bCs(HeXOJYc*pQo8i}4?rA}vcw&{1V`@%+ew7ZFy0N^X$hkVC zys=AjLzc`vn_xIEVei=3ut$zX!wVLZeO2Lwf)9U{;;2cmtM8ubuU~n-yY=%t<_4rh zCqL3In(F_(A}`1opSt85FF6tkF}+Y(C>jqh)y^O?saS6|fWBJzWCKS-E(*>A~e(tCF$eUeC`}Zfs!w zy$05eNXa7EmCFwOKTN%2aAaNE1{&M8ZBA?_6Wg|J+ty@aO>En?CllK?`t zp^V}zO>l&iQwt=I?1kI$M{3cNBum5~sj{u;Po4P}J(`Q%C-_b&+(KSBMnyk{Q95)< z`;wT!)lPloBK%w*7zpp(8U)&(AYbsGBK63;(7FD&82VVQ6_$d`*q<57UmZ{0k2*T< z52MtWJQ^U^E%WV=tIt!HR#j+Csl+4Zk;?-z{PXuny$DQ6{w#bYl3CaE(;=dTVeRK*-vvZ0lSCjBO6uh7RA!Z#2UEsg;d_80 zxao%gwbrGs*2`4U!5)tT3@&UX6`d;wwcDFkXmP=s#5roj9|%!fCxMo&c3@f1A-r;; z=9vDmY>T_J2BX=k-S?C6h|j9U0zcJEOH;;*^9jGY`gQrjgzZMw89JV`L-OSw&$dJ= zrkd_&HdTcVdWOve}>@p?k!Y>sb4tsfBH|WPX1x zV_Ak}rhUnw`^qqV#ZyFc_1X}m+Qqs&o3WhVZ&l%%8WyK}NeaW98fWLvM8??`9PKu_ z7M^v7vNiJ>SKl;|$N4Px|J1*fMmo`znIZy`&*;DdIEz zHi{1f)rD!~n>Fsoks%PzYXx4rFpCAVo10YwAbd|-rmtMJ$T3Yk==6f~#ojWRBP^p$ z&W59YF#U&s|M?FAkNHkcjE=;W3c&7Ul8EbJGVz^IY9{#e|H9ukb6U*`0#x4sqK|Cf zX|0l6YijVq2Ig%BHv#$%98l{H}m%({F%`+B~R|@ zuJir`f83kt`za!Y2h|KQU)7R~nL0BveJf1A>;2qh1#m`1*bW&EY5DGW-lyfjkWO}a z4Fvjf{}Z5{3TVIH;p(T-RwQG^3OC$9P){}?^|^{&>9aU+S2=B4b64E9`ziEw#6&zpVf9e_FTu0j706b;CVQc56QTv`-0kEp0fbh4=n@U-qN56eZSf^nB zfvU)_k(1h2qVsZVW{!Hz2(&9nv|g(!%Qgfjpm>JZRW2x-3o}Yj{cu=}YRhw5OUwKm zDH5{i5!eO>0#PXa3qWuFt*og0oe!+9GWYJztSaO6LTs_s29JFy*Km5(pTaL&Y5UWzO&YLoITuP-Ak0pC7JCa z?yntkACY6w5*W8{bOfK!RD-_F8#Ch+X>>5b9wmt9{TBgUROvL4e4cwsvwBdBLu8OkBrp`-JS>Tv?(cT~!G^G^c<%mi}qq+sYnZvj~0%WKKqk#rE>Yi7go zF!Da5z`G5OTjz*uDrXSfROrhh)@&8@sN9VZ<3PCC(B0p&u#gz{*_~-)&})vA)t{|c zBD;k=_dJi6y3y_nFYu%0&YYKvtkfKJcx*wnO5GF+ja90k<=8rYC+oz)!FndDA$PJVSRt^@gIZ7CGQ-L+!z_#!}K0M%$Ry`T z&V2L!jeR@kT`z@H=sfz%o_Qi0bT8j3zWVi`X~D}X4|jy&#)P2mzE5$NJU9WtH`-JE zZ`V>o!P`5${MPVXpKVc|SH+gQ%AaF@>swgQO@;>8C))3;$WBxKjbN>DI9f0>5)7_a zl24;Bf?1U7sG)b3rJ4thSfJH~Bq1@8Z7i4)nk-M#N#8KMF%Qm!1+Ix=MDqG#Z5F9& zaOX%>6A(FhuVaI+mIr`4pMU=9-|ob^b8dr(Fgw1 zALeCx0>+e|8uyW1Sn%$YU4a--E>M+w5m^rm-P~B2Q|EPl8fH3%0EmfmDXxU5zlxtf zfw9a>pMah+_=xW4Z$4TGHfCAApD|GSZ@%pGn8%k1$UxH=3;T%sU7>t6a&$}0wQlO$ z3}mNya${#HIpq+1o}{~{yNI1D1PD-1)H`=zfU|@RIZUb}fWjofw(o3&VVRxKWw-zA z`|yv{T+52;b}Kj<#V-r-s3PAsC&l(Ql0m6UmiKP2lH$= zJ~z%!o$O`nndc}~%7Lj`&u%i3Y))xp5G>TC74`mwnuOlD8{2)uM#i)QhkFDR?`f+5gaml)ZvdO)re^?V_}?|tZc^CUMBB)`AM{Hc zxHcUW*WTicIzlodfflU=ZIFQJkVk|!x-Ln9-({Sf&IH5F+aaCTguY}l9+B^P3gK!=XvK7HEXEjo489HCl1{jp9y8{6s!t*UNG~^j_OP?8^8fxyrw4JT5b9zbd+} z4x!k*`}9i*Z3G>w`#zT?b{9IhK%R4A{K{8O@ccE-196GzBfckrupyrSe#w0x0#v95 zu$5q{F@W)`q0fj;BZ|`n+%19%TRE2_?JYR~GG?yCW#vzr@c1?QZw1<4q6va%{zBn? z#MfJ{eApV7E=9LQ!ARxs%h3<4gjG46OhQY%dA;mAv&l1Q(M$wjX#vjM1A>pYee!!+ zPp?uJ_g~nmN7_4(%>2{S`S(!Y*e$G@8pt6yXepPuWAJZ0YIqvPMM zD?xv_!w$7RZ%yhpuLSX;$*~^@OPqo|$~a6z$wQ&L+cO zqMAGU5mPLQ^xb%^$>vR&<16zsH!7%k*=IwQpInDeeLeEd=jGUydOLFeaJFt7+caA& zrEKMW_3k09+Okbl()<=Crc!JiGKPts4bEG8LOmG23d+2HA0j$EJbW-bBvy!jhx)0*pZ#zxD`XGEJ(Yz983 zkG+kA9h2ncPs5JRhZ@#9HNjuLK9!S!*o_-m!si-FpfQI4BOo$6Ha2vi@$I;8t@K?7 zj2?Eg%1yYBFQk3`!M7$%t|w6R&(6LGrNdx<`_-Q9Sw3pOs-Mk0%A?z5j9%4sedCU< zf}z=Gh)J;JNQf`2R?g4kk0bi&yXs|N0#xfOD4!At`R_iq&j6Z|spY<@4?V3b z?BQ%;UP)kzSp4V1W!^JPYHjOh=eKBGNN+Ok$PzDu=3YdoB{v9h7dvkWoqS4)b?dNG zVZRZ0w-lbW(;HGuuz8Z6i%Oe#=@mz3kSIlJp4Wczlq>WuUhcd4d4qEnxYtoDO8{Tj2h8Wbyaeg3Nz}zhBp9prqNfdc+f!Fz8Y#7H=C=&KT2yYfP&vF;+gq1C_Tei}Lb1XqJ_k($*Ts)e)V?OZj<5gS0@;(1_>cdvc}=|m$Yfo==v{e!92jxDdQ2qZTkkWAnH(M!! zux%1qLm(E(A$b~0Nt#d6pN+eKzujURtw@E=l-5oh?b<-nC)VrH2+ zh~bJ>@4@J2Xe%<~b2h><@`=pm@Kk&ARZ#~R0Y8?fTW5f*w*oMy^@M%31ukC$f9?89 z-&VUeyc(aKWkHO7khj_@|`d=9>G@E~;*$|df;je8rza(`Qr19-c$O)Ska7qWI zcc)Nah5blDA>m;iBUDYf5BN|2{(_qSD`lV>{couZ>3=|FB6g;4D?={>BqJR>+qHtc zRa7?0<-u8VskM9hVHEX^@#=7L-u1h6JKixM+asvx3!qK+?dasU{2pj`X#Exl3aIPr zvX$jlJL*D=O?@XDf(eqw$?hXM(>Uia^{cL`n3@(nw&0~aksVJvTzRfy8fR7&@}$wqy7S)j$L88P5vru+ONjB!tZKy~15UACMb;tj+SXJZYT8h8 zkP=G+!Pr8quHPa>7{8CW{+)<1ElwwOu7^}b(la*tM^lUp?^9<4mOXs+;L+1V9f@n@={bxb+)OQ{~)zb{U;sSx$35my1HvW^Rj;FBss2c z2}JU|us`C&=J=HQum;jZu}fn;^$$rCIzjL1n=a#m{3fsxLX%?ufwM@Wiz;jOW_Su3 z??--+4sl9a)S)EE|CBv^k>EF&2Gg^E0AYav>2Z%fT#9swpPWHCo`T!JX45Pq76Ic7@DIY=L9Qx*sBLi>DncaksipBsq7eTLs%;Ml*XC4$?} zDG(t6TezU!1y3#LS45<_F}ddD7jiHbc0*{e*oKFv|%vKUH~QRe=u%}qEDGF4CF zMNH94xf~OD4Sn!c9XwN7sA)r0n~TS{6geoQQo-{$$bUh#F%7bRCTOO#Y|%;?(@5#RZuPm0J2w1V6 z8WbbH-Tw-?qnESjOZri67-kiho@ze(Rr=Wr1Ez>)Re z{X!5x6ugw0TAV5XLiMwzIHF$}V$0Cqk(QS#w5EhVKH@OXEA}?66W)NPslKzYPP(2t zGq%nG;$sQJa4|JqPL~P=*@ws*1yCj}djUtP4ikd-L?)v9?(~yg%R42{uTUCrz{cXI z)#WnYw^J$9q&UTfrPp5C!)GF}b8WaiscY)DsC!;3M&sh2!DY4xmcs34Wf3K8^7_Ef zdTr29t9OWoxz;x3vH7v~tbJ2q16b%98@g?sM~f%``%nZhWCN_d*sN{>H>Oi}!+m_# z+FKeD7`fSGs(m*C8brd#mi`5QibS8p7rJT=5~Y{}xTUpa4sT$GiCox?1WEXF?`C1# z-g8&(mMltHBlm+cXPh9?(`g~U%#12^W#mjZZ?z{hs1u+jL~>|OkJ=gd0f1f4)GtE)a)}7VCLq55I3p1(mEl^V zu-(}oZl&NC@u)cZpR~T*P%9uBSg@3Y_rM$cap?ZzI;-y0zoIe|(S{Zhc1{RQH3w>& zppkAm$f!Jqek+{Nc`txox$myH#A;*@`NsMMrDL;+qdR?<@*2=?N_i8jvF(^w-*yzf zIQw*b;*-vm@F?8QLA=cd?bA3Qb-xzE?n?)O)Q zG?$5_(GMepxAeZ>o}>jz-dA}>y> zcb!*m_+euQK|~85oR$zsClY6ZPU?1*8E9SCCB=Gf&%|d?zy5O%BH$lT$hy7ict&6; zDv$UrBb-t}0(1{WHJuD35qsCBnk`R zmd|5mL=7hBbWW7kZr4;_p33dasZ%%?rkcsE2gYo1R?gOeo?`%QB~^aIq)#gj=Z`X@ zK!dD4OTujSyJ%bpktQ_Ef*vg(IN@wkmP!M}UMLsupn)S8ZA-1jyj(zEIY1(&%KZW9 z7oYcKY2j1Y;PxrtKUIH}Mv9}Ux5{K_^o+`7=CnsF(0tnFs0F>0UYQutNIg``-wjv(tO~9`mz#H+Jejlpe5Nd{>uOv;!xOUun|%3r_ag1Do#J+p^7u`)39k} zI8sFkkc%5?a_Oo3Ty4VX;R(@-jM{~&T7$#8@IyxT z2J;5V?ptSx#?ac>a2JpsR~rY~=stQC7C4gH?urtMlxHuDeDa#=o#I#Oj`6V;8HQ05Caj;+3yuJkQ@c0$ZF zR{VA7%db(C55D{3N(-z(9*;~HuJ+Q1&S2CCqM}cLHVNLAJC<*$xI3XVVW=`<7%)IL zZN#~8Tf^c==8rtf1CCO9%b!R$ILD**%;0YSl^H^v$vgY9hP-?D%S(xJyP1Jiq|)3; zISTk6^a9+^Yqj^}n%CaG1+qmECYz?=)&SM{S6x@Glyiwsef9I7!ge7)D9155to`d0 zV&D2NmUiK?TCC?HCgAv(cLXuX;K@1TkuV~fHOXsW6wuYuoo>PVeUuGU%pBb54pPQF z%v$@SF{D={>0z6qSxtY>&kz3zR2!8WvH~L}3#G5a9L6LdE7o&YQ2wXopP&rpVLzJ; znUhXL@YGUec~w!LHtx&HOY=uEV&prnG#NJP!+<#VRKWJJh)(B6HQ7HHM)>=GC$D~N z1A@m>F253Js~_>cY}GV7nhVsI-Q{PB8iWH^@nUBYRN74cm7p0G6VBwr|4ja{5E*d+ zUY_;6*7JLF-YNXPp4nZnTr7!430=@o5`Sg_gs1MGW4Ymo$x&LFGby%T6@Ci12t7Z$ z3Zt4P4>fvs?#3>PvnP%sgTSJMyC6T$3W2m-jN5Xm>5fys zYQZWv=bx?p7skz2;!dapu?4@+u3+5UGxepXu6ER>+KSC)y0&QJ4u` z5N#ZX=0fLp36Z-3H5sxDVPbM;NRbrVR~2IfWt z0(bCH6`vKnFs!s8BW7Rn6K{bx%i5}HRvqSu-qd7qQA&M1u2}cOOz`LnU{e~9q?inN z(WV0o?KaNe^J3eO!X?)Y*$4#A5pd?2S*;tKR3h1`h243Z2lblon6%68c=csA=htoqIzyV^0GhQ7@h5@Y*TmEh zLdYMX|KSXG)gu5(k;@EQIi4F@?>QuqwSHceNwztF!A|No0c6Y0UQ}xSh?0ui5J3UBWo==KzAb zgj6NA2uwAOJJK1(oeH9q#Lz0u-Z1Glj9+&D?Igh(E|XdD8XAoS*{;@=bJ{K@ye-twNwA3#T^>!RWoz|(Ej@d?P)*$xAJ^u|7H ze5`bT8LQJj`1+oZ6}{k-$8NYXGap!A2rb_6*Wt~AZFUdql3n{slmHsyX6D~@@IS@@ zbF&JnpMY1d`Tt12iw6Khl~hX_=o+i=POU)2cl4kvy}Jofk^cm9hQuFymu^g2a#LC} zgtBP&IRZHFNBBHbCKSfuF&l@h7W_$t=Ehn#27Jzv$sbmSO)yO4^|ak2G|h_Q6HBp6 zWt+nt#Euve3z%&7+m$m9Hn9Vfz-GTVyIP;bAni2Qbp_vw0%rO)7gbqBrA1`5SUfE_~EP}7z>GqNN+-WG%rT`?%E|T1uLToz70+C*<*^n zQQ$c;>cJSSSxW?;YFIM38j5$HKCIcha|NzhGrcKO67D zS*hldqi}(tTeo}?!O(=+XMB<|kTd0{wJ7cb>%aQ-Z?}EcskLou+{Ko{#8fF%f9Y8L z$Tc&;#XX`wBZ@{5)}?@qo^GVO-Sx)wZqT_8i(Xs<%NEe2WJQErv{_JtT=Gk@HLe^{ z=sm3rG5t~Baw{iWKnGR}cW%C^UB#X}PnQ;^?lcGSk!cKLepIZNlQee@v;+j7Pju6L zF|Amvav#bK+vK+5TGJBgmwI;e%gx!r`mV@+%n3Z!!}FY*qmCZd;kz4rOIUh zb4@@O^mkMANjB9hEi|*hVQ3~1sb?&Ro5mLBCT!whT~s7*0r_`ZAM*R6#pll>xZg?P zHz5nQyC#vk#5nuMrqpvR5dT>%B2O~di}!5^-4hKYoLGf9MfA`#nWsBJXmE>t+@GBE zBA|jFXz#uK`8m>BKw(;2G!kn|S*>d-;_;Df!5UIB%1EuQUjF|Mw7;Db)hS+J7D->= zjPkL}JRQeH0`r?|N8bS05)%n-{dLA1yGQN5=HlrYP%oxj`6EtIJPU;4%+tC0s7oCL zbo3ZFUd`jW*eWgr;h25`cB)Z0J=p`?c@>abTri|UjE10b`Z5X3-D#4e_g)oMHNOM) z>E*G!LyYjPrU1JfzJL!!o;Tw9{!@^dDZsb2(Aj^rp*=8$<(@y{b58KrOt0);jAvMA zW>;^!o13Iyh0n^fp=HXY+%J@U2oq#C!P%tNV;nvtCs>ol2Hb9`l+@DxAfU6(SRp&< zts`q<_%3rT6TxW5#5UptGL0JsHemf?m}euVuT21ed(juEcUK;xYE=|p^duxb8i=T% z00q=$CP$(T4%k+BU0(VBGD2<;`ybWOI7I9d$g&UUEpN@{LIB_guNmQdAIm9rCzY#g zUdw(gdo`{C5WHO(np8QKBnv+kM*#>55nrnSQ}Q`>Cs-b-Wy>J+0t-U^=_KS)_)D21 z!Jt{f-(;(szEFa26_ySU5Oxv#$S`lU32hBR0!crZ6K;~5>2z)iZ>|Nv1%Jt!gLNYu zCgZuJn+^fiZ-Rkv0&EnsEnJ##5Y26UqK|J6*BaN_Z>8oJ1ihrU->Km4p3La{^$9wA82b2ajbm(1H(aJId@-DJ7DRJuJnB7gsw!aV`SCnQo}&1W24j51~H7E?gw{Vi5_L(I03D`vYQ0%=Yx z3`OFk92CZMz<@be$~y@glqW^hA3{opOZ`rO5)ss7^n3d0ErQc)^wKdnZPR)ObqDAd zPlg|rvsGF5`5vAucN!LVt&d~NjJmn;6EMQV@Slc zGJ(?$R*PnU7c2P%3(`i5J*3MrFzzZpR@etYM=&JMy#^*K;+6u+RGUR+Ki#5BwG;)7 zu{G5Yw`SRRfC@2lkX9pX(J0r5eZ-8Hp)^TMZSh=m*b~ISJyd&cvMv@*lwVlNArna+ z_60}^m`Z^JCnnXrH@03ur86tqjP$5mlD}{^Jta~;m8fCAO=0|Va6fnvjBp|<(1<7_zj%w8e7mywz;(0p zeC#lM>sXGVP=M@f!bB+ToBU(b6h`F=f*z$aU2aTAy)EXpxKMQ}@@IcdBasKWQCoxkfp67&EpYyx*e93PQlq3C%PeR2@#+%GLVCUTEIu= zoq|W5F&qy$K7h^ul~e)BIfd>soUe1CkNQgR)(ssdXdy`^K6f2k^2{pQyY7Z61=j3_ z1U^9pnxbVuYOw$z>DK}eG`vH4wr-PRyABoCtO1$ue-WqKI?u))__ROK3iK+HSoxno zF(HR8XpAwtAr8z9npR&Otfr* z2&|b|n$fGUYtQe{KFJY0rzkWiP1vk80#izIiuAr%Mfy2uH0}H;=&6O9A&`cm(+12s zkF377@jA%EJCB{POzbAOf9*(CJy|RJbGp>6uRPvop<^p%^ot)8{*6mAZQ5&o+XiJB z^p(NFScezZT%GqJNi7h~%aKuCShH)4=@lX}$DB{0WaJVNcY#^Hg=dje6NLgR98QV8 zg0DvmnIcH!D7F4NpHKQNZzUSe_A4CZ%W$nP04BX3L^%R?P6aKB_@Ifj2e*=?lJeHf zEd8&5IgF4<7?!2byx-Pe?X%IDX!|wOYIdzEZKPDmtC*wuTQ(QF@!x|! zeneg_msc7qfAlIEW0q1U#JjmD;{LOcY)>a)hi!tEUJ3P=%IcMWD$v~Pizm}VA;J-A z)6l4$ie*sQR27mAR{c<7L*}{{6IfqH6|Z8@n1?=P{|%j|#zWfvT;N)+j95dQrLB$| zWU@n>&LC#nDwtyr4NKFDz4Ek@kHhMeSeO)@)($V#3?7_F^jM~lBKnD<-!R2h`Jwp^ zwV=Dv0N%ieR^zP|WkTr*WF<(k>1T9fB#4#FpV7n^ye0*T^(KX`Gx(wB|pIDmgI7%VIUnSE`Y*=my zg-pk4PnrnhHl6^gV-&9cfDPiTR;%4B#UH{$1^7b0#-)s?T?(tsQIt*}ax{%-5m|MM z_gK%Ne888%T}O5uzj~d{x)j~8U}7*%>^`dl)B#xC%OAPA=%JqUTzr+OK(UB!hp+g& zI$^Bzw+0{l&NXpRB(8_W-t4|{#A74dL0v4J&g}$vQZjB(YN%BeJL#jyCDX8j%q17< z!L+m*IQ4nvc|3_4+73$PEaB6~*XFyMX+sZQ3HpMp*`{@#9T`EJFi&U;%?W6{rn3AX zLkrTLR)Or&?9O}O#mSvK%@^vxNSCeCm2f^S-_(Ds+&H4Y-{=$s^A|Il20EJX^|aDI z1WVoubf7s{t+cgGwkC;eXMLp99O!<>kHHVm8W5om$%A$fBKZb3Zg-LBQAdrFnwNEQze5hOjM{B*Hl|?$I_3mz2vr5r~)m^Nuga8kq*LE z=^kHcUqif$MYq+{R^}+W{oE|r(Tw`Kx^yh3C+X*Te?aEGfkHrOU7ZP~$4N`c5E`wIa>w(G z-5So#zn+Ff){>x1#=am;ZQbn-BcZ=$*+$-Es?1^DI|Smn0vb(II7aeGXUn~Zs=B7t z?h+L3NwmM;Bq8!{FD?Vj6dX|ArYTqCVx3+9VY?WI?*3_4aYx}+w}b49zgdgvPDbD~ z1Zs+z#80TJ9TqNSS2ZQ`L|*)pg8nCUJewCj;vi>vQgKC!Mw{k14$N?Z79shFSD^BsL5^FQCQ^}SAoT0Tp_)57*?p9?KXrn1=` zz-T!}t)OliTyUZ|V#o8ywvRS<53#hY=8T(3v!Yqsn%Boc2dLeU1W1liy}UwxmlnIZ zLh58Jyr@f$@<}^0#vWWqb*<}Pi%kk0b;Bqjft3fXz4FNNPqz3og~6oU0~NuXC=H2pJRKTF zFsRr1FP9S1w8%k?zl5+NU&`k>04w7YnS~s=NDkVjo`rq(rdEDw^*kd?E~^9|R)1Rc zb6ArUv#9t9atqQHgK_T{XIMvbU`~7PqlWH$lrR2Y+kI-H2wulNx6?r(OX8?N37DHW z;m9S(a{`d3Z?1?xUOcBSAGvk1M=P)-)k_N2fo=QD1?lYYESlD=zcy^;_?5u)kHCc=J>X)fx z60&em%8Jft;Al+7Pi)FD|6U%ttVs=5uNa$*|WWCko@ZN_FXD33k zZIZ|1kP@{2@{R0L{rD}at+we6BIZrfMK?bH)oj0xO->mer$kC{4O$~{>fZy3rZX_$ zc*6l7hmznE=O@wwRn{0O95ehnJZPdM+A_3PZz_C;{^WriR zHV8{^faj&eMp+TCnVZ7#2+{&cttUAOJv1ok!z!1P!13_!*XKstXi2?cD30Ey9v0zy ztTpRFMdVt4RBh!L`Z)4^Cv{<|s2P{?n?^vb=MOzPS}*4MIJhoP43Bv|^Hx~+ zWI>Fc0WH2PNDI`lj3V)^inZ7N+ST{!0cl?wbtoY6a7PWF{ta+z<>c}!2Ec9e0B|_= zhGNS;+1}=BQC5}XoG=dFuh$O7(ME-d{`&!azy9w5!82E@R^J3ZkvhuR14Pi8P>Pe{ z?}9JQ!hArSCwzeQSd8>41`rM*XW?;+M&~0;%MlB}If#wK;0fzo&)=5Tiu?!}Noa7a z=45ilay`6wKsZAtYH)4h5>A=`f#t-2A=$ZdS(NCt_1v%nrVe{-gsl{I*Iw#sBrD>6 zla-#(9Bc(RI^O6^4O2ia=lL?~msUHWMy{KJBus?pqFs`3woLMsw6%FY1UqrSDdN*W zBP;1_oJ9G^SqDrZg)UgsfqN&&B-7EzZ>ni4io3^9 z1@rKz;omZhsZXbkM849T+uul5>SSU}2tW<#(HQFI2txG??0l~oXxb|!erGZO*!Rz0 z)c?2#dwz#qUTtk}^A1}Mi}|WF!mWrqFjOl$kNfo|Wv-V`(7mrOxyDAv)t#UBkHS>C zjG{7LUv0styt&~AbjM4JcV9GQ>v*!M$bGt!%MxDG*91?WEc#9xsQH+2P^YZZ2NHs0-j%{iKuB+*!ioFSp{!DrYswFK8)h{?@63gORt{cyQ-(s zPoMD7v#2m8dx#mkbj9CEy(PW5kO!lF)WS6+lawjGk)u8!)9b7%>&KG(>?eGdhcLjq z4PZiBomw|`6#H`+oC$*qD5_wzM3eu;Mz6aN{`~SRn8KnSWh+nWr0y0+h*0D~kbexH zc+f9KWzO^Qb;SNqYrBox7^|$0(XA!zrM!UFjQux6v$kVCD@V)ShSpz=Z;+=$3L$G< zSoc`s19Ke5xt`-D?1UtyKV7VXnx4kdIek>%2OKF;cqmE+10Hxb*i_b`sdM%sMJa-4 zku7U%`!cBL)IPyG{Fx(m_OVoh2!{syipZ`Kmxpg%2lWqjYi;EtxD~MdU;U$S=|CLM z=IDtoOqKP--DcHR@k&`vQ?2WG(v-N5qEJ*K9iX~hpQ0(pMBPAlgd61+G^w5Rj^Zv) z`pVn@bu`!~#0x^15#l_-XaLt~=6DnG`jbbH)qLZd0^bjPsUn(#wEnj_qjNhh`KTip z9AN_CFp@gRbNpRduk!-LhYCmI7sQ%UsQNU!T8BxkwPMmEg$9=}xc^g_i^9tSc-D{* z3@27yPme25B@MHb$H9DLz4KMTbx9a}9T@TfOyj?}0l(~8B$*v75Tx63C_t`f)}lYW z|7Z0x+5RIXIAy>57b?gRRNNEBw~#U$G@yO;+iqhuBsIy3@{0 zE~4*OPft~N-Piq-SA`W6&zo^vyG#ybXus|+#yE$f%n2HEnfaAj5`A2Tzqw&}4B^N+ z#`?hU@Q(Rf`b$qFcW9&*Y%h)MTCuI6X?LI0k?u^V>npPFuJ8`VK@T`bN(oPdqjr)V z`9n{S<#gUAx%M&6%G^Qz!sRQpYt|kWvf34C@?*v3ETJ08W9XjmVEg@j2n64;zzakV z-B)A}y;rVKfP-B)Jg_0|ylp#~oC-i{3iFlGEy zw1EYD*WTQH@G%|d3FDoKAz(~%X|9x zXdsU&9R2u{;N$Y%woNkh!+90|x_ObTvt>$FPVDst>sVJ^XLZ0=PIk(N#!=ha8sfi^ z8EF}9cYDoC%?YR5YMQHqeG3=a<_80D-~uoCwg*?3jLrkAF?Jbh`V-Q0f8DYO@|U%iJ#PH>FEJBY%Xry%C?Df8-X z|IK{Y37l$Q$zoAuJ#Gt{FS3E#vsIw^U39EeXZfS0joKG@hcx^x zSwbHjt=Gv}Tw8RcKB!#=ScTQ{!_+PTrv>IR`uYl?;u(Y$sMM(u^QHPZ6HwwxF(oPN z0b#ndxm*ek_TZ{trq>;Vz`9Y3gCS+X2YNV+}1uaiCk-ua13^_r~%L=T}G!(@u zqQ`S%swd|6o>zFJn_{VpVV7Jv^CINkRv*|&^DZTY9@$`< z@vxT>($0|k&a+tZW<5yhS1J@pA36j`Hdo<>=6Or`Pw$Ijs`|dwQ$&cK{*j+HZ8PtP zdwse1!(Pts!*r|JT+dW&m8FH%S)<2F+rE-#Hqv@3iT{ zj|;{XGP(wDdYIx-rzJx>~1S9s|F6el17!Zv!T3`@k@pH#lDKx)~MH= zvsCap+-5Rc1KXf<8PdE|IooZA#T^$1g2}}_MW_|W^NCS}IErI)$|b3YrJ=7Qc3uP6 zKj5UPIQ2mD$pzYI6f24l4&>uKLYeN$guwqe-b#tL^}b!h?xG={UI1T{PwMl$R?S?hS$>${+e$kyg zfC~eiiK)2xr9Tmw-J#o9V7@EFKN+)(4m*m#T}U2hVj6~=37-P>n9l=~65g@jMfyBb zR*@qDH@#DzjmG5v&g}lTf;EM-fal~HH1LR(|0;kTZYw2T9`pZr7XiLc&pKX)FK%iE z{hd$yqNNFlFl-F)nE!%F5pEI_ubH6eyoIuWsoMg(g}#Sp!q2)-3a~r$h@z4j&`~<` z=}xscwY9&3gtsyaj0l3%;@Yk;7c=Wl9g<*;P`b)oz%a>KAnT=6zFgUXD<+zkl&twG5Ka;C@U4YBSsCKNz-9{ZTE{)&h5gu@tTJyu-~eLVwMT2IId%vO~3Hh_zfHN>S%zC!%y z*&6_iZRaqzIT!mEKeW|?4NbpD-X~#TD{t}yRAFq&=$Z2QP*U@-x5k2XiJ&PrEi4J! zMqgIk09+wYo_pze!fWt*oXe$UmDEitusLo)kn@Q1Rw(2vV^aOfYSM%yTQ0?rzK~i8 z4mp7z#&W>s%EMDgH{^Q{Q&O8!<{IpM#=@O@aQE9sZ$J`P;EtTIfvnO(|4Bv> z{X5O);l_D22R^3SC`I1Yy(~l$&8_;h%NzF?$#Ulak0-wxFXQ9@V0t9O>JjIDc&VG8 zyq9rsZyg~s!~6?cx6?!cN=Lz*ghhqYg3Cq0_U^qR#s341Kytt6;!gnm#FZ}NOwA(h zM(PF*J4PeqyT}C|>>gfZ%KHK{Xy*(vtdo;JlyAP z62B%J+}{(bqcG9gXx)!0QY!3Bkk21QtXQ{GUeh*tu_h+_Wf`&31x#yL%5vjrQJ-|? zQl`qh7f~dW50YD7Ypp(A%7yt}&io>E^7nfl>B;2QsRE|^Fu?VZ`(2s^_5caNr*q;y z!j}`_vR4IxOzMq%8ih!L8cA+8L|Ei&^pGPe*)wWxt)Y51%56d)QJlgUcADt~vMYg2=Z&%OPi^R4s71?54UMCXU;%r_E^nLwE zY;`%Y+A7&ZTO^yfKsP94BIz&!qz?>V8K5--Arf*7`vd7QLM zDZ|?ppCk{ly6)q9F@}p|1Xn3(tg=kBkMgEUN=8{yjG=_@0vKSqB!Hn(36UE)2sn_V z1L$FPUou24ytPqss!^T=_)>hXKFW>jftZ^m|?4WT7sv6Pzrg@ zt<#eFzg<0FuLRn@v`&k4d7Tnyi!&oF(D(IApw;CUYOC1G_CZzx3867Y2z5$@P`lpQ za&viS%canA`oCR0U#}Eepx^6~LJM_yoliK$Q@B;l_mkeI0%j=ZE%jo#JWbpibU%w2VzwZ-eu!;g}=cB+v zG$J=j6!jwPu^RS)5`v-S``% zUn&tP^7`uct2he*@!V`rM(}<|8yW;3X)i_VbpCAE;9@4g%eyOg${tot#_flqmxE*3=2%ziA z1}f^2iuBFmpDFmb^xLX(C?x75J0T4CFKUYPW4-$l=et`HUIP*_{X&o20XTrzuLA75 zC_ui8{P?bj;Vf=Q^;VS7j19-o3lVf>>W!i?-C$Tq9f|1#ap;wVdU!Bou^XdY)qo8l z(^LnTiRRL@PtC4!Xlvz5U2z~u6XJ1%Dgc}48t3v^{U!XgEQR0#dgbNNCnuZ4e?FAcx@_ zUUT7pfkqXy)(L(vb;MKIc=B4Ad3YeDL$fkLQ=XQ9+nyqt!VY z`^hl!{p?5E`lAWSru^1PY7&)CNGo4}#KlyG*PIjg5;;+b*}X{N!Z$J_sYwqxkS`cE zME4?I#zjGhWRa3I@tgIQ420D^mFuqHu}(q{7)N3k^@Bg*Kp)7BQJs*`jc_sPp&{hI zg}k?__IM$C1eu2K<9)Vj=732bxawgKI`x1CZ<1&A?wh?O!;Vy}dhVt^rKCJETFODC zl`n&$qOVF%g7QqaqHrw!sp+Gz_|tDxza|G^nNJtTV4d;fApf`wa_0lew*!WdPd@VF z^#!S1okQpXp8(YFBZr;F-ZT=Xya_`R@>%3pllZI;#6@w})m^zz&w3R3j9+7TBxdhp zw)Q>)5QI^Wkn3MVwLKS%JAUE{+(HUMloaquEeYkM)5<5nm+N(ch9Exfr?FcqL6H_ z)d=zVKi7}{OLU}1*f4pBN*(?PL7&Q#Tdwy?l~0u_e+YvB`SGbvWkL)fRhfj8i6CV0 zd6W0v|DFLChkV}e;}-b~{ObwCWa0hFA1-=V54U__ z%`xa1Ae=FD6OGsZAU&ZzxYEleJ-S&yV%$^;8!EF`OPIaRIk5BL2&EZ_UvUL1yrm42 z@9EAbosX2mMn)Xs0U4o(iJ!y_V*Yd;lYI{|M41c(C4mSA6^XNCJRl2UzmFYtb{`?1 z?dw)+ajW~^=}*@3H}~p0J72+9p;?oQ;Hx?$iQjj=*1pz~pVaR=U%{1$hU(@YmjQC> zoT4A#2wj1%;OZ9j#J_xqHTef`A*23FzoRY|y;nMag$=0%d z*jNhF4aZm<%D~tX2FBMi*u*f+>7BHI$yE4oRc^n~Ala(<$N9>CS)i`0)f`A2=%SY- z42IB`f{uiGZ;dDdFk*xbMHtHXo@+u-w7#wAfERB>$p+Df4kICQ z>)_1uOd*#%6z^I5?lrnMmuNbpZ?YA7c*NHX|C0+dL>U!IC%(xl9lX}22<2fsh^A72 zw}b**rTGP|Mq(c?I_-*x`9{5?DA1^PTJ^iSOyuGcL`CorpVbj$*b!?qW(1W$P{a?Z zGIzuWQ0atX@k>c6j9tGI+ri_-uMFZsU0=F>6dYVO09<`~|IO_2ruTKy(RYJ-T7KBw zs7{VQ-fm)7%MY}h)7xo|u$$A{Nr%>^GPgB_xozUXM4;yYI_MV_KdN=hLIBoxly?{h z2t{X;g4YYu(KnU#sCjJ4x90Iw^6lUxmT!$`Dc>4fC*PLS<4y1DW6QS%dir?sZJ|y+ zu6$cYS07Kl&EMN~$hY}>J5jzVgLm^NB}jT+x!*Y0Z?^aIU8~*H zTD6AELYd9=tleRBSqB%=VUNJjRYpPOCWG9xhh6i4%RdA|52G-3q*G2h%}$DRo$18z zVJwD^o~0OGeb%7O?D3}e^|8h9N^${hR3{%-46in4&gS%XqF_^=E6sz{?tC!a?%auY z=fh{YJFh;Y&Sv&_)BF0^yYou&FvbbSvfH^+o|2TJ>Bkn5bw^N z$K9PbV{=}8qM6O<@uPF-QnTP>7QJ%LYPM2~zBS#VZ^w)N;JGb&6}wsEV_(`jQnk92 zBd;*|j_@E9!FY_sK{Len?tv&LanlwG3=gu59zb~?HJhnvZB93>t$147d|KmTAh(ze z^|V7DK-a|#6CZlu>daVZ(h-YZiHQ=bm^dElxh|3RvNuPuQjB>uBioG#PUg|`klfu1 zPvXIlP!{<<^0XVf@M8BN!#4Iq$QY6y;u1&aBy@ydTe7-B zRFf>E#i06}h=^i}f4MvmAp+0`C=7|pyg(c$;_|@7j@Z2=N2c2W4)po!Jp@L?#eIz2 z^d^t}8|Y#8RP0D~U2W@TY1Tyy&82$a;XP!|P$u{|9l_ueWqgDFBL8`) z6uaeDPAhrhRlZe$Xzk5i5F#qFWt@KclKtX^q$(fOu8+OFZ>O>%mJp+(QN&=+LvevI zK{`pRA>Yqb9uwh_k9>zjK0_gLs{pE58e+;wcz>mhZo=QbhvKUDE@CJgVIQe9FM9Fv z)^AC8&D#8YCr{)*@eyCA;FY{CgRP#=OLA?g-Yn-A_lRi44BAg^fbR0(IL5wBLb)1faLuCQ=|^OSN6zAlhbJO zN+v$iTg~?reA2ma9E|9f-S5ml_r%U*h%Yr7wNYF^-x;D1ypM56c2Yj35=4lVD0)Nn z2+>_Y_(JO47qs?0jlWG(VUedhJ72*$4B?0`KJXR1M$`$hOiIN6k~tK0;9|m8+RmFjfdB|w8M(-&@qlyks zGcurX=yO3lMC)?LGzJe|c z_)$C#CjAP2A0kmK$v8QD%D*wGL}J3qoI6zv@jeR?lC=@lg?z(%g0u?lshmWU`)>f> zDD(u;z^@nYKtQlBrK995W5$eSrM@WYQL0ah5}YwXWke;WBKWEYsifGs*`ScgB4=zI6~_o4!%5?$F){uV z4}21$51N)5R|x)kEAD~ZO`Yh4>M24XsS&CeJeAK*Ydi#e3nNrDo~H#b$hbbC9dj{t zLY;rX-DLQjGtzzloohHibU)kJUL9N_xy5I1fvbx*XRqJC zsgGQl6=B3D2>UXjjyPrc;8x_1^E^E=h1oPRQR5S{7SW+pE2)HsOO&kPH9z&ML~QR> zG_|pvDj@^oPUu}#jzbPEh8AUs-L%&j_QAzJ-Xyu&%=wm`xzq{T7VJcl=xzFM3mPx@ z-9XIK^xu5z5BRy^!vOnt^M4x4R%$+PW`?i7VU9chR}pjE89vrp{a9P2-@k5CRR(bW z`EAo$rgGsfbi{Ftj!W2>O+8pmGB~DI%N`%Zb(@xtA?l$scl5aYsy;fYQ!C}$Df_Sz z{+;sM%jKl8AC(8Y_mRhN;Khe?Dg`FxCmadHGD-k`)*R`V3Va>Re)Jq_ozN2AOZ1F0uocoT|-;zy8wa!Vxf;VtuG+pKtddCeem;HS{qvh zw?pj7le{dAYNRkVGV-dVoU-TY4*)@Ye7*;6r1q3#ZmI$$=R-z~ESITdFDRY1Qsop$ zCHIq4Cr;$xcM@LL&O#FTYJC6j$*m9GQX26&6lZpo!IbzQ2=NVHM?V=`>(0*a`~>?G zAp_6A3-ZJvd0Ak1se0Ef=(np1d!4c_f zGg_+G$r+u0Nz8I;1~dKFT?vR=m#QSr3ug6@6-Y`F#gnx2rqT6`k4J**Wusfl4M6 zuEu$4HOmUymqTll_2YR{Ggc=2SoOq zcrS=TU#rLU6cb-otM2#3QXvtGGhK_Lv(O^*PF~ac&Q7xq&iInh3UDB|jgZ`6nb8}j zso<+sA!L7YOqTlV(nVWMo40Ff~2OAcRui)MgD~y;fEz(MxoP z8LjWMB<&I|2gKwnoZ9f!!hyoALSneMrg~K^IKglp5jpetV3NMnciMIETY7Q7AG^5I zvzb=+RbyVpA?QQw@#z~8SLQxqL@8{P@DVdk6|>4^p_}d98+c6GvsrS5;Ss6=&h&sU zhruuiyu=S+XA0mpQLWe5hX( zEJg*DWoO)E?U#RnXT|A4^PWAjFo`f+h*cCAxWyZjeVrDG39UNTkC#O~*dIj$AwLm_aNgK)E91$f?CJbmh5Z4o>V&lv zp`D!?_*8(E7g`bNHj}0ddkQ9o|9~6G?4;xdg*O=8?u!=G$}jARqUm^RpnB*#JpswX zdxE;@?npN?skJErIk9-wPF%tnMo_pWZFt(~a?htg1&7e80%BrUP3$*UfQ0!>xWbdu z7dbuyj{`nk`yo&U_C5@?_$Zj+x>y^QnER0 zV94%;ce8My6U=z#0|eN_9B5ucJoJtXwBym?u538f56_a6w#lNH~LMv zS_!3Xc^m0Gfn(aqs(IgwI1)Y(F=SX=mqBRAjj-^d-CE zGa-+3*+aS5H_N{l=Xi!H)T3*!tDB`pr z;ie$PRjuZR_cea2R&!A=8mPr~ft0U~cr2#%ZcIDnR*Jxj7X=-9@xt`3C|Xav$rmpQ zDe~flUQqd|mGdaIU&7+07u@O2&VP$@wVdxDL^K*9rGr;hmZ|9*0aI^n^$KPk_^Sr13 z4Qc`S{-HtxF z*w7dM|8qa=L1%w&XQx#MzxrnAP}$1YvYxLbmD)WOQd^ql9Kh$r$aHbQA5i;l61W2)lX zmZIDs#D2D_SsdKNJLVyFr4Js(W6mbcm(LbPOQ{C=0Hh~?aMDYUKnnn8NC-8HadLtR z3t2pGBD`!j2eHl*q+11uIrTlA{a?x|Qs3EW*TIEs)`>#lTXrP^3;__w%jEHNn#|HQb{`<{2PHnJ1uwt<^iUx|Q_b%*6-qn}jbvs<^Ot)hKJhm8v0m4@aIZhuFiuC_>4FAMue; zo5l@86?lgo07oQ}g>fk(Asob(G~}w1p=>Dhtq}X#hCCuJqN-w!I%HInszPqulKO}) z1c_B#M9Kask(LN#EEx0Y4iN2~inXS`@XCXfet;u%3I;(-+>u>;&wF?ZK2TZN7?ZPjj>K4b3o$*!2_e;^E0d)4 zot^x?s#N;U5INVWrF8Z$K51g7^3VV}_o+4bJq72#N!gNi>yQgMK9(lF3F5eV_Ah-( z<-nZyD7Lo=CmJYJcK{l`vS`{c|T%Luw3ai zhm3k6>|9v0V!Ma}s?teD42u@2Evv@GYRaRl8izeX3T~0-=|wBL!XjCjDAxa^y~=d; zRcQDveAGr7p8$%5(nNqbQGU%h+;Vvs`!dC`KvaKT>-dFd`5i+(#WzTmqi6AXOh|?# zkO)DFQz5o$dL0_hgf*U?RQRE(tc6iqKF05Q8|(4?H?W;cJ{P)3myz>9=bsu&?s|?W`aJlKBsl?V^PRNdd+=GEBv)To9Ot| z7Mw&eownfY$Ke}$YH|ES9ZuAFB5hRGCl__Pk!vMU!(iW>I|E`8;CM&``^Wo4wkeGj zjT4cn{xJN$gdaNJHe$b**rAx+%JZ2RNph|L@?b&Z;_1y8j+KA)%zt>aWNsh#1=E{h<5P z_q$I&-UsLuR4VfLE#^hKmbmJ;0o`AG@BS1Bt^N)WqTRvlK6REk1V$m|I+Go|xQmA)|xBgJXR&VPF)p&H{tg53Hjq(dC& z&G-B_dAJ`+pM16502psx+FdiZJ1eDfoT3IR_hT)=5^bm0iEj{M0*83~Oe!3R>N`8H1=v#|A1O~#eK5b_F9xF8wZu_% z{?C=1#h${_iD1$tCSoq&Zw2v&jRFfkT^1?W$bpPwOei8Bgho;xH+`af)m6HrLajsEq0w-`t|o>p5`{t^VR*@@ zO;Au*+Ot%aiJMmaZXfwZUdS3$e8KN^_V?dOjDKng2T-3+D%J?J#Sd(caVT)%jp-#b7w(%%=KZ|{ zGOi-e7reiQ1XoaN+5;0d+Oxil9Y-XdF+ogLC)WDg>eX-ZH;k7V?s+dgqo>^-{tT`tx99@tkaBb5wrnX)O>u=w6g$!yHFU362s zY1NPNsca00mTg_NuqSr~8XHaYj(h_{Od?v$IRH8#q0}%q=C^LZ?gJ#J?op+b+~w4t z`Gu*@eac%mPLZiq2gw!AfJo)ljTfj|x0>~)+Wg`i2}%`ikm~IGEl^oYip3M}nBX80 zri;80kT5%mU{wi&zWWaRDjXnHGBM*LH_dvh^Y@*dI0H2HbGNI)Qyp9)CkipU2STqU zj2~lMI2__TC}4D-s@L`d;@0#ZHR<8Dzh{^pbz!(TnlMd^`p(W773vNu9Kv6`h*Ln1 z8QO~%V3&l_I414r+5<}%#dDO<3v{olaELSv26E$J1OzDL69AwOuwRp|gK>`Qekv83 z@Y+-f^JE&bdO(9Dc1;C@>f9c{bTdjvA%o8$1&~tWU?F#8*pz<{&M(gmwRV1H%tYp~tx*n8Hh(&9y!D-IuccZ< zQ_oECP`FBU&Tr8jp|83;m$|-RIxXoH*cFknO79Z;x5WmRc#wO19cg`-hjb|H&!$m8 zER8DsS(RCUXF@?4TzsOVT@VJ zj7U{^)vzUt5o;H{-13UefJBA^^w1D`{e=6CT!u22u(`MDE~WD_OV&`kgkVh>h)F`R zRI>Y6ocUpqjVz8+-QUY3A&q_DN+%;u4ky#jre-Jh07|Ako}XA3B#ud9J*Awri$^H| zvB-#@df<{l-xYy^8A;KX#Vv&L>+ey}Jggtq8?xp=qu!L)PU?+%TiV$_5QZHj%)-j6 z;&ewFp?7NFN*R#rMr0yIFm#4-?n2H4$x{rfrILG0dS&S+3niq-`V*`4sY$$q+i2*Aa?XCg*!#QMVyp471Vi+1)f)aD(^659Mj~}2c;uP@@X-z z#Gb4ctpK=Jc_3KunsRZFS%jp&1gG^cjst1zlg5k;-+i&p#A*>arm1uzNHT-y;B~_W z8b%%x**;Lcu%N+J!lMveCtcBYTCLQn#t!>@$wH9@Nia46EX0F>^yW;YsjqTvFg!wa z@TuK8XdaHmz4~9}I3iy7PW++LIoFO`$H&Lbqt@ZxMrlo-&QHPNqK`d9_fv`c_Qn6} ze^BDj@r|+i-)7_J@Ti&A|F#-OM~7Db`xrkDI{;L=UF-)Dqm@(eX=ew3FI9O}=g~=D zPa>Q7jR#>8sqp<;JSMHP58BTcRC*Ei*jvAH3YyjAY9cNwr=U_xln=F9#du8OmCC8i zaGP8oVz=Nbc8LFqcq9}q_JbbWZB#+4*{p(Ar&9&ZJ+o<9B>+^;lHq{=l4@^iKF8$> ze=Zh-6?N}hH9y$&u>NNJImMvRT_Ac+qtULC$h(5^x{4XQN%F_81RPxtQq~QTN1)rK zkuZ+-Bd;ssTNxY$U5XsyyYxXHT>tv=b0zY{uK&4GH5>FGr7!>Ux$?K{#m|*$EXl+J z@n4@S-#)b)^vj;e+_0N{F-31RWfFCcn=>(2Cx`OC^qhHkg!wXSHW~#pDJ^&4LQo?$ zhjd78#S%1MqYQ~yjCLbE-*E5`bkCP|B|DzXV}dWfnAb4#(2wG{pX(2Dix!n=MXx9E z@0T65vMR{MEVEE510S)sZsipCk>|-PAq<8e3Go;X^Z+YZr_|uI>a-8qos+}P!Ew7K z?t0k2PWoO6ppQHnGp}CF6$d&)^bwB`iJ0+4d5_Q7574mS&IdHz=k2BL*luz(#g0d-ox-32@>*k zk=>0dgAlvtgr<8rtek?j(HQT-q4D=!rkD5p?_ATE&vdJi{OvOo-as$AJkl#nktqnU zKlmsYezWL$dQl{uH&{GLAB66hG50{-;!VdFzg5RK7Rq3-agDW-#&WA@1n*u$2G0rh zh3GV9zrv{@ZoDUA*W+7>FLY9s)H5IXmovl*2xWcDC!|uzUi^{x>=OSen(Q?GmvLXj zc4yuZ*GgXz!+#HhF=XdwCjK7pUz$KncD(Xo|15KZ#4n7Y9SvZ3?O{K|OMUElX<^WA zRzb6QTm{YJ6VA`g;}pyH`FC=Hls9ZPBU@?lo#CXtmlX_Ud?YsCSHz1(ewwbzAXQE_ zEy%^*xXZ;k4k1J3!dyGIJs8&l;%21ra^!Vu^kr|4ld{oB3D?x7$MHj#q7WmRenfNp zkC30R)1Z#8DbyF@sghY0O4c5XSrYN~Ml;mrLRE=VHB4}!GEqwRN#ofN`h002pXKC^ z7P(Rl$`9(vdxG6fMmaVaMcf_FC^D+^<&-?4cbAE5A|Cvpwkvoukk6_mY6W$A&HBVg z&qPumPA934TUJtANo^(dWJ%3sG`tyf$qfp{;kjH^%Nr_1X;-ZlO#I~uZ}Veia=a=u z%`ZP!pu_M@BG~5;=#;^{xCEOc>|Z4$wvygT`cqH|Q0O3^MLuQ=3;g3%NqilN1ePQ6 zo2@BN=S?f~t<1MFe>$1JDaZ1v%tZ9p|9q}|I;eHN)L+CB9qfZm>goJA<8iRWNv8@< zPWXTI&UjpLDa|F|0x|@4ed5;+zIN{JK!|8Se2NV7g@qf>4{o@~!6PW7edBp)8`@du zaXh8E!9ovUp@)SYlc5K^8B8(>C}aMkDiywcY6`^h0vtA;kRqYTDd1^KbVeBTtW38u z9hAxRR&z?3-nKH`%5*E!C(Cq^(rZqCy;-Dux%37vUxLOW7JxKm*DJT*L8?nzf*&FF zOlEr0vb*#&4m}L%D=lXqlC|4|VNQk!chz^uABgPI%|_}vDtdet;hET#3!D_?^)M6* zCjAx8@+pD9~NAoIn(eKz5i>RGKJd}ijl6p9qTdxsy^36$Hn(vp)&Re!O^Btb%W&``^R`skk z4vS19os~fe4)VZO)WHp*izD7{Cw)E7M`NIy!@ zm|D~P`sEP!bBjF|?r+r~O8xj^e|GcmMK6#phgz_Q&>;+wTd;-6jjEO1Zn_#NIMFDQ znR#=O9aA9<;ens@{SMyYjQHwBBIR<19DGP>OX3tWcTja`$vj=WMaO&Rr!Tqg-^06c z2d!kihv?ryW~GaaXSq)B8)WbUq$syor69^ZE+tX!avqGApW%{z{;-~2nzsBR4!O_I zxj+U5%T!!{hi>9t@0g8Fz~?EFb1J=YzRV2+9^Yhk}g-v~t2+sMXZ_p@u($Zu(PJxhEy7MN zT7lX?Cr=xf$^A0BFD1Y$bH@U&%>rIv zcY<0hlUgWsrIj}Ee&AX*ioDxOpk?G~@_5Hsw+Ow5ehBy%AZDh?)h zU4|A6*tj@>#X;Sx^IzvdbUC*k>uQh?`FLp05%FIc zVKG%^swAYS*)U^@N^#I}1_!kcGC(M22R_W>Anb}rUM=NvY$HKYXPmqa-~g$v9~okG zj!*@MOOz~b;1l{Yg02wJ={s_-Fv#K%FIyxa(I@2wPuaaUj^PAyHym@)>)71DbE~i* zmRS;DQarQq)YzEjhW`{O3Pw@88#6zXm`5d^#GPW`|g$Y*kav z7z)Mw;Ud}uA)+X}Nxqd=q)_N%CRa`VW{FAym>W4|A@&F9?+TBdc-xt{PsJ0gf(Yyk z7tOGUN(bL%<93TNj<{FeJw17TH0q(Sc=AXjJZ0ob2WLc-fY5Lz5Vc< zr)ZMg`th*<#}6|}G&^MlQ}!{f5}A^thS+tHzZ?kv%da2p%;*~>$&<3PrWoy~zrU0QHFg@ zR8gb4e6rt6KiSh~bV*$HH+dE(#=|am0`t&v1-$#?lBY8pg7L)W^Km!UpXRgMW`c*$ zrDp{85V%nD12{UTq7wVZ5c;%_LJ?b>O=Zhzk*CB|R#o}R4j69{D_(Ir`Q59WN3fXy zPkYRaq0xqz*;+bjVUfF0v(Y-pxEvkNZe3fUrL4H-%+#i`{6TQWGtKxzWn4&`p&8DG zA3we;KO0sYlucIpeY$WJhs$(rhKpi1|)Wb7FjE>X#|>Df{&o`^!C* zXZ0!bpUcPk|5NI9(OrN;)K%_XvUdD+)I*_<7@`~t2sxey1hYzaDcFaEvOQ1(qUPSx ztI8Bea{j?FDM$v_sOup{k?(}}0qZ&_WRfPbo_ewLFL5&)3kfo!{}l=u?qdfs1m5uP z-Iob?jt2xOx^OTE(Ew&k;}wYY9rbPAWxf>o{f>O6LY*NR!QW6wRl)c}d6`QMxnHZ0 z)41lnXj3`o#Vs>53OvZLKZu1@#lw^tYtj^9u@7w_oH$S7)`}5@@j%h`qtU!6-eWhb z*N{PQnFbd|rtkvjA^h_snfYkqt7h^4e@7@3Uf@Ad-wnbyQ&$J*F0(V5f>L)RE7Mc1 zPo7RaU#Y(^+nGS)v#jKdiTk&17p5{6n@D3KPAjH9_8l*B(OF>%Z#^>~^sZqxHi|rk z3w*oueECH$8PiobN#c`T>CiVZY4;G)VHFgeWcKEq;)3911_xpu>Hp|HR?>HKgW=bT zA65%Uj@GylQS?zZREQLx=&CYniSkqFEQ>39qFw14dnotDPAxytlay<=U3(k9_U2o4 zYg%v9s&~@sG9a#9mE$_N*=sfLDLZkY<{e0vv`XXTiVH8%Nt(=WiN1fqwcFZ^wJZMp zvZEG`40wnUd0ga$+}4`feOuRYA-9#(YXNqyg+5BXtVq(;r!MonsgD!lcQfv%(l-BS zjgsFx#_vt@k9NbnV>p?&jNhBCWRWj=4E&5}qV!2$F&jQK8#cevIxzn*g4m8+v%k78 zDf0bEM5s@t7m{15wsIp=&QEVY{6AnH#0S9io?nTCe$fbls@x<;w4HB%S{uH@-}SIRs0An#$8+C7^*<=_$9<#GXfzHF4#dBW zMkDj@VdJQC^q1zr;ZdvEY&08(e`&Nj2d$&OfX0%>IQA!^42FMcEc~tP;I`yPadx+h z{UBn-@m8}{S8T;NLZ~7pjJx@ zO5_*D>L!9OZ|`x4pRP33sDf6rSp}_5rwX!VAhd8hONIyj3!Hf<%mx`GCv`gP5g57! zIy&*MlHo#6AL2mA&*=TPF#Tj@;3FpVODBiT1AQ%o!SG`ovZqKWZ4rqMwSpN^3aeAl z>>Rh62Zx=*!**NQLvpr?N&tP7+RxL$RfQQlOQP1xRC=byodH@OvU}3Im{kgUQXb5n3ukX-?-Wg4X-NBo zp!P9g?PY=O<3rm^tP1Jfe7wKVXCHt@y;J4?(rBdoYxcx-0EB;n{1t$?eTj4fAhyxJ z{0}e<4f+95+*bav$N$;;?B~ourT!cRwgjWRd^~k|FRRpInVohgQxI!jsdtv5Ui^FNoLOdzEOS#)cP`MN=v-fxd%!&ENI9(qu0BJ6Y$$$FCW7*8 z1BE)HT*d+HyZ8pX5iC)t6lWGF%q=h`yFhtj-Y{bLbE~N& zg1*R{l7BoukIOr-_I?*~Qa%qsc_xA}xd@Wxi`9;s3ciO(LT?%%p8?{7 zP~@H#oIOM_m`y~QwFxdT5E+$l}cIROlfclg;- znEtq=e{wi|bIG@DK>b%Op=04~F}lU*g?+XdeG@VIy0z8E^TX)&WEedzRqG?c?DAr# z7K1+<44#0M0J7nd)QS#?{Kde^ELde`9R2VB_+YcH^P7)pbdC8NQDdIZ+BsxHi)mJi zY4Qc-*NzV?uGwB(lhHl?NJ4<1Gn88r=?R6RTLqX=FhVYdexhtFJ<6Gs&in7gHR3tBzsBsp6 zMJFc~65yj9Gkx*aX<1X ztE_&`kC2avkR>D@DAF{#C*sXN<(N%NYzc58k}dN>)9+I$}J0gEb^iz+9hIekeR_q(1f z7jGH~=YV&6a{$YUU0DokZYsir5~)m?7-LiL)vspmCF~%eCd)!AYyHgi`PpKs9pe*b zsjMW0eXd$otEF$`>SbjsW|=j!jJ);vx>*x*Xct$Um|Q<=QU$Hd3Mr^0F}aS`q)J+u zT3P@YchaRY##huTE74F=q+x7bt#PFbOR9dPXCY0tX&|z2>FZ-3YmdI!>S7cZttgwa zG37tZ3B6RZAzuijycEdTVj!jEK;|z9QabP%7X!VS&mHk zk93`mbO{in2uQZtazP=GNncHW<}m%4^jFQIAnCFo>B1mpX^^4`D=H9DR3b#Du=rVL z&OJ}6Z|Ij)xsZdOe(YWULjM&@pj6S2om~Bg()u6j#=hf4l%Y`nT~mM8eBvTHvGj-d z8DIb7;P9x^$khKhXdks~{g21^;jCU*$74+G&cX^FQ`YY&tKLzp-=3**$Fy}j%Byx{ zOD7eS%bowU%SoJg)b@t-^g`hoa#``cKm07*05M-SgtWF))y9Q3-zHXr-v$56&5QB zlaUrF35!*P6BLAFlzf%Iiw4+Fj!~7PdecbNTBZUlQ2>^!|Hdl+N>zWwiobDczv(M* zH$e@+o53y?TDFI-C??zOzWmSU3YV!}Cx~JVnwY__x7c^dEfseJxDl5>SE?i8qL-g5 z*!9ro%C}E{>vnmYFD8UDy;&AR;fCxOqo@z$|_c56{xY2F6LEO zY5jdt+U2Vn$6if{Y(#a{4nZ$}7G54{(&#@u##Im3IWX zpE{#2VArJTUM2O)w{#1YAS3|_8P4jLQ!37N0HEU~0HC%7Ko$T^20%0#?Fyvh5$JZn zFBcpRbvfq>bhE4B4!x%gjk>C09|^nvj!=ZUp6D&-yk)_gGESb1bV@`|;YgeFjh z;oTyY5DBn&aM2BR&|m-axe^k>`tcT={0A36Bf}lHiY+?Z`w5 zyq(=_3tvB-)&BM6wEp+K&y{aukc`nm8A!Nz2aR^-?BH`S9lpl&+JYirL6HST78GrS z5pEt(bZ9}51x3>t;bb}kjyVKM2oZE8N{|jl1^z@FVTiX1Va5~`3q5nw}BpqEWh(l&2HOUJLr15|BD4%D*{_TWKxY;3)q-`TA1}5U{=loHV@7^ zvN+4)tXXlEkJv2EzCm3Sh6)rI3LcMqy+LePfAQ5N&NT|_FTVCY==;d4zxe7v z5A%O|Vg1EaKHnUi#GTLp#Ye0gKq1p~U)-)p1**H_W9P5``M&-FQ^;UYfAO{c;2zmQ!7Z$uAQ4E(Zlr0KLN(sOm0m52mE4d7>z<^Tk6cV07`mFHHfx?VFTD;4mqF9b z)9a;i(X5P zSl!5~3{h4$Vs#@{H=_S8^2=!+%%K}Owz?6k8<|%(5(6B=WU@SXle5)4RV=eK{m0`^ z!;@EaYy!5lHaBZ?d+NAyM>U&I+Ic81=Tdg2Gx!zd<{Xn3t2is?q@0}NGje9KmuFHt zPRhkOJ`-mu4`+PU%&17~r{#lnP(V{&h1lWwIl8K(0X>J1~E#4vd+lfZCn`a&~# zVJdZDDsf?@X{1gx(=cqU6lAw{mh&HZ@SEX@@GQ+zs|uyE0h1P-8yXA{C|(}WAp!Q zYW_c~%~-xR!>0AK%8aLw)^CEkKrE0ckG2~ZhGR4Mr8;b*aGu3cwfdZ}fbrrA6Z6?j zd=|4<%%Xl)or!N=O?u3%lR-&|lvqKsy@dwTEFH|c^64!$obt)BDf85qwjM5S9MZCg z@OdG^&4>fCneZ&s+LlasOUZrrl=Vn9%iVIYfmJ10;9!A6{I^YB2P|%Q?#b&mFJ0Y} zhB7uYUAn^-$UH|NWATd3D`)Y_hU1lW<&~R1k^{QSj-Ywm)N%8UTqKLz@L(029nMCo zSg50?(?T7a+s$H~&A~dm>qzeQ%LO3*?jE>74kO`iV{Z@q9qg_#!&|;Xi$>MT+*FIB zvI+CEvIIJdOwlSro|ivE1?seUyqroVXvIn0=9r$- zv#P9a%jY4LHmqchX2|6>iHkO7l2(|}Z3Qeeetu@i<2Gr2Hc1YxAOp$@re>V@u{qqH zmj=V;aI+eWC#Au#8Q81_!)9Px8va-**sw{~mP+$k)eD<|@2qvoTBn{mwzGNEY#y~I zl1EM6FUg-aQ|7d%@iQ_1Ti-*7?e`+z_0TLy-;AH}`QMsHjl*pIw`Swuz~+B@j32?_ z6&Fz_#9|i{a#zf@WVvj5Gr2=$lbEH1WBTBp$?#FIfmPE_DCVryzJ5 zP$GYry=gYs1O=zKLGoDk^8WafUf|040%+r(DiMnzp5!AsYqg#sYc0+V`Hy8!`mg0TfZ&9&iJG>D zZmXo%Q=*ek*h*>jfv_jFu1LhXq~E3YzOy2V{Sq-i`rz{3cb*8%P#AV$5O^55E27z( zFa&4v3r~b*BjV!z>In8;HUC7Y6(HZm{$NGKI+v|K5nA0s%vQ{fzeCJQ_;C^%w>W%R zIQ+N(hqn%6>cqCQV#68@=`~h{>NlDa0NG!c(Cu{cgtK z#VW|pZ+89+zk^n__~;@(Ta3)tPc`g2UgV-P;gg&8Ov+JpQGyon%YV`g^Q{W_U&|n~ z`c*hpeN&6s^yet^=*aqXm@jJ=iHNCagoO2eJ_+>emYX7U7?;Q!G%XyxkvH4oa>|Kl-!#`u4{j%V9?f>=+GO$D{dp46obGW4LF z`O3v~RXq3dvsC7k_=L>f>_T2339nf_^lJFa`h%E1kIZ1tOS8HF#Xb-=3(=mKigf|P z{SGlxYKO%&70-`+Z^Cdk>ODEN>jK34C1StyFYkS4dNsd%{@GJbsaKz)GIjx~X$}5j z3jay))B;nmbYN;7BsPPCEHJge)B@9OAfE-M7MQMqau%3wJ215#0}~D0*_p0_+*EJ9 zEjwq#DzIuNba;OD?J4cQYGKTm0ARfRx7}$sJK6aER;Od_zmM@#X#X{HWmvPXHT!N9 zO3^Y(cx815m~}%G5V~+K0NrlN>ua(0=Q0TYg!$1qAXeYD8jiDuVDo1tmRlxkOgPeWyT;^MrSw4LWc%+vB#OgXg#ng-CXbb{!J= z3k1&=o;$Lwt4;55bCO-?PFN67LcK>ie2izmj%B|kFvzoo0!!M)o{RHD8R2}wZ2p7= z=r_WYt}b-@y=?u-63=D{ScH6$f9;c7e?`>$Rkr?AiATa<2>q3tq{N63I!VozB;zE zbt;!Z#mDx+nzRGiy=NnonKl^Q?ma7~;x_F)*8e0&L>mzPC``0l#zp!s;oCLW`J+~P z{_u&f6n+bTW!y1m;l{1c%N8GSY_R#t*kZDP{74w|o))ZF;-)o~(bO-*W|Q_HyZda$ z5yu9)+1aXV&sqbztbOgd zImQ-J&xXSPxQhH;ZUDlC(l_NcH3 z_3Tl>9u+o$g7&Cjj|v+=Lwi(s#zzGkSv*DbYq{I64K7|naPe$j)lW6P*o=5A3No$< zGCieG<7wl6iO)jfc_^%fkp%V`J-{NnCFT8QLx2tPnZG zcd&=LC=5wRyNodCP0L}QI-@Upd!Tm@b~WQvK|tJTj#(zJfrp2HxHI$Ih3Y(pT%v(D zd<5VXKPG|z@!#}tpN;e7rn6#mthd)QiF2LIi`&G6!m}J?Sn*ji;YOp&)>tFzGYIeG zAoAZ4icr@@0UItAOfJk!96SCG?;Lo|J7>Y;+WDp38pkisAZBuiDIaA~&}IoF6SB+vK*Rp6n$X{6eDrvV-w_$xCr;HB&` z=#vn6cZ4WI?jAVrG@gxs+jO2i7^^~g29bt^ zJbSX)JbSVRo;|TVyD>aFR!j2?BJzrP_GGhp_GAq_dt!NZLwL4hWA}2b30-$qC2FgL z52sV$NcLOAbioguitWx^7hFJAY_+f~_V_Gz_}qQoGwc=lee7d8M6R#~YSA~7%zGrZ z`Zewo5HZi3Whzg3~O{^V;J^itQM#FoSbqJz6=rLt8fXPIBU}6T*v{;q|Mo}(MVJHWf=?L z$%P+jqWwX7%OcmqxNxFBNN<~?iIQlUbo@j*rYILvvxdN;DdT?zn1v`HGnf7oKjY(n zTJ7dxw*FtUbpd0P@=z3$Mq=HF8JZHbIA`Ox#E(nu|D$C@*B)TFJ)ZaSNxak0hz?51Nkou{hr z-HLO@v#;`3#6c=~(~_!Q;yIhNc3~3QOtmwU%qOc&@cU%))8@{Z z5W9%#v8Ns=R8xQL73LI#tl9 zpB$MoOg_?TA5}rKby5Wf4gSpWNx?Jx_0EC(uTejVZJIF*%q)lVJS;HPQn9!+mu+f9 zBC&Oj0*}3*`Y|3>P5{Yh1~3|=p;2W7@AT|68|I1{VPD@cFVi8p%^{r%8*)rRJog<( zGBisA^NIIRB&}C%OwSIT3_Bc8;D&=-(?f&@LqVA4aU-~^q}k*!`O{}8yn$YJsbl&9 zWyhY`coa#40Q-ZFlC_#e*VD^BHTxPc3N#290Sw)-6o`|zc+*A8qp-0M`hy9LCFv@@ z<@jU-?_NU&&k6R$ay3>;g_B6!cu!pMEr<9%au{+esb@a&FJ~wk5X$=aF1;nGi$4;d zUE)7Qlby!@GVX^c|Bkp;`idC-dl-x%dkkPB&r1`C$&Ob(?4M`o z8p!d6*w0=OFBA!B$A_*sn{>PUVB!-Ob)6uJT>f0Cj);q1ey(8GOQIe3_P`6i^3s!#>~^97byQD1V(@dN*(kaB zBXpP`Vl#C+M16rpDw(;b42jT{#+aFpw>(b0%3-Gpj4Y6b+uHfB;8%t{{HFj+dD%>{ zKk$&S@x=E8*P847*j(@8?)ddic=szWiVbQ|| zqKD%J(Zhj754tgn9xQs84?WOmQ~(=BXhg#M?t3&kyMfr_pX&DR8KN~>*U*wn|z{RZC#UsxMk60DQwt$r8;esgT$f6Y8m_;cT zrOby?c6WC~|Y`d*JUtG_CQ6M+O3W@K??DZ$vY^{oQQKUq7AI{`KXw{`b9-Z*VMB zJOGOST{sPOnuVVMpjtMqr-5|E*tO{`wR!VC*Dz=psKvvArxnu&%5 z8oIHK0~*%JWIKSySe=B4B<4^{$k#ttsz9kGcthKV8mx8#SlD1;gIbFl02{K-sWZZc zwuKG4F$)_kY?v7~EXSu)DkUD(2wltxXNc!=*?E|eos%+oHCuVviCq`@m(l|}OG{Z% zwb4oXC=&&zdjf9;1sj-N0r@*4cvo~4`vq4^B8UaAN$CQlNXMEx#YQ`?h>-kv;k?*1qiQT#H5o2SJj`Y8z z)}1M^98|_h46&A)9P^LCVfVy@v;&I|06ZDs znQdEaX_*Vb6{(34gUIokbl?u&;f(0$Mb5QkUvn6I4^}&y$VqKnSI3Ra z)5UvpyayMEynA>z?ue92Mj?vTq?ZRjhgNdI!yXy@04d4`8w*jB?sqI}a+i$(O=QZX zuXM;~ZOy&Rt@``?v_lUMie#*~{tn$BFI^bun+0#{1aDa<)p@~N%Yrux-sS>tjRtrD zwlAO96Gwq+R#rJFDd@L?DN299g3GDEW$fc?@#V7c<+@Ru#g-OZt_54>T#V)gm+kT3 zlF$Ab{g|yxqHnJddxm&0WS509)`c_1m8 z9}oL)U_rS)@mv&&Jz@&X-OMJPPKEy!n&Xf2pc_FK`-2>y6nNVJ+ju0(__rJ5z-3$& z)$o`?j>Oif)+vIaGt6COFqGR&7N`w!Tbtg&ARYDy3|*S{WZ}m$QMhnhroVKik0(uZ zF7UHpru7LbLFOb#mwA+9AU=~bR{8EbaMmLc0~fO7CwLzh2L0@R_VZT(rT~JUkv9U) z5QYr6oEb@gTrOhu^K1wie=;DqD3rH8C8uNXOMpViNcd%U$XGy6_xA^w4WnM&A*21I zwr`E7=ZdcF)xlfwjUJ@P1;htzh(Huj79uzTpRS`Gsu}Nz!^5NxI%G78e9Z3mhsYb% z#0cp=3lZ8MLG16V3x#d(nN;CVGl+dTcHT2%`0-cjoW?)g1GU(y;z3Q;z$5n*{1n1I z^dW#A@dp&U2tYs)1Ee3{a$^PoP833xd^1agsp*}0Zw~|^`2#sjbofUSf)NQ3z`m3$ znE0IjLvmZNjDU;#eH2pA56N1lr3wNML5jc)Mz?}2uYNxJ@Y9=jzx-4mxtx&vTD~x@ zng|i4iClo?0{f&3<;rgs_B{Q_9tcrDC~x(?oPJPnAtxatz!4i#6ruup=7;l1Kjad} z=#c7#=n=mYck8BAzuQOt{+{Sfr2uP@jx}dxeMn(xv}pk zxR5L9-a3?**}bO(LU=7&N=uZYlJD7q@8$*>tBX>5zxHM<(DMOcx;35`lqWZZggR=N zvO9)huOWlrGA*VI@9Y90mfGe&?Zj7&g8GTrF08I@lpSxTt`5*$1{xZoH||IZgC+HY zQ&q@|YmlcakSDJN*tr&ZBlWV>8>Ya7?sM#4WP>6V`5baT^eJ(~!p)?jil zUNjpKv&s0WF4H=Q0ZpZMFCuNL*}I7vqwQ&>GR0>11smPoS#n4D}6?IgTgLlFl5M8 zI*s@a7O9>&Adt@mz54GQoqw)W#WP$cz5HD1VHbC$KTAOfoeb79zcz%v>#gQ%^P&;) zA#f>Lr7O`Z%}4Hc2vYQD&3#s1(jH7#I$kv%i=_f z6ZQS|10G{P(fGOIIuovTlRSNe-9HvQ#> z3WLDIk2ugX`Mudu(PG2wL+m}`D1Q{6n{JS=W*cPj_yXj+*dHu=9Otsx=?3z8Lon@%Q;U@@%4up7}&i(#JwhRvEZ=fSXt7QYs{Yt&KF3w+p3&`fM zd`^q)CMf^XBbCZNhq&mkFF#j)`Nzt~&Ox6H8Jm^{o1*p8GiR~Vf>^0hf|bZ>J=PXN zO4Iq=Srnz4u_)^CEynY}s3QxbER32AqlTC=G6>>Uu_UpmB^g$h0BTqECATGBO<;`I-+ zP-0VnmPiJ5$-DhpQO!cG1tC|f46v*ptd#KFa{^U1Z^ZM?8?l)=A_}uFWYR9^oG$x- zOFppsfw+k76((Xx!t^JnW0*Pqf2PQP`7 znlt|2hV|&p>CG^vXT3>OT`cfj(H$4=6 z#lh#x3pd|zWsHq4H#=-&b2g)&7N~78P_tp<7N>1Z*!c1S#n&GhzN}dAwFiGMDct)p z;=7j>(QQMxElk`P_+?>Y7A9IWXale<8r&8%xOf2eq9NBC9%H?%sOk+@ZrLzu8%90B zJIW#%i)3sNw1qMo24%J~2>KaEJ#RP=vSH3kheS5ad6Pkp4Rf|(&QBrCxjei%7ubAY z>&(ydGgJKc^)anEWE2n|`HW6o34TKScjM@=nTh{CIBePa?~n3hqJKw-vMI{}RDna>`BTF_Jn+}QecP&fpl9cA!KEM;xRAuM zmW@?fHa6slY>)Cb4XnOztD4)$?)b8WHk%4!N@SA7Xuh&H>{PpM}Kp zP*@v59}ZCXxl#pO^u7FCaZ%ur`w{Y4t_G%jW`0ve+%kC0JGPRl0E-R&hIr8k(PfYP zH`(kg!=BOCd9h)KDDbcYDO&oRU&xj((M)qbql5HfgI6(n%O3fqY<8Amzh1+zq2rKf z`Mj=1eg$QpRlL*Ln#h$HA9LU?L?hy_fZ<<@hG*pWpB{DLGssXs z@>VeTOV#v@!~b~=dp$?m@y(U&`TPs-rOpt!k%!y|hxjh$g8k*^iuk}HsY+By zln0h`&x0)eLn1pEQ^@4DC@gox5^qE_h2 z8$n1Mb;C!Fu-!cf?TSz{Ko|RioNi$Ot%7K9B}Rh<#(4`dTk1G}mkqAOO0dW{!)1>% zoMD_Z8U!4Ky=1p_>6LPxrjmRO&$I*A^xM&IBK@C@gE2OKYQ=q);5yu3-{mt6xAks z0vY5B5R&$u%o&=HZr)A^8%X!e>GW~RQrWISR%uivh?}rK`}@?Yi%PO=Y%sS(^z(E; zs+;Pv)^Dv^^{aMu|I3~HROdf#@%+aIxHpz?|I0`Pc~`V1)gh6}=k$nuPWWuFO5v2q zMYk)(Rk!=_3=JzS~F+`(` zZg$3G^cc>V>>hqV9|R5QpVbYox49do7j_!a<(QrIcD*u=t1YRAbH1cHQBuK}3dZQ4 z>6&+Ta}31wV3g5Dn?d^YIHH4Fe|)gT%^=7Nlj@1G@I6@weXXG|?mF3#)w z`|a|+8pce~E2W>`zPrzF-}v*}Q$K&vEjs)8cR2j*n~#5c`UKd0AUIaG6YZRlM9Q2B z#9jl$J~;eW;nZ&nr&h7i>P-azzfu6$MjlcaU_?eN8F$~2(assBA?}5w+ZzkQXTkyl z1Q_qe!!f5m4oyk@gj`j^8fs3|ej?||0yX=SHD6Kbb#Vu@?1B5v-a!vk?lW{l3Np@_50Ebz(rYi}FV}&_k{E@BjB>Jz-4r`Inz@5(Jc= zDrS$yUvs-~GjK-827obSUbg~8HzF%oEf)Rd_Z{#5{?Tdtb!}E9YGY5eQv#y0590LC z+x;M?g5`eD4K^#Spnlkp>ow)472rY@;Hm&u1-Oczbr0Kx*8yDH72v7>*8O-rU{94j2QH2gXPpA+ix}83hg_C>j z#NT8K=IO;B9}=9~Z@#Z!+%3VlA7g(1z^DRn6@XLk?;pTT?Dt&%O~`VbV-sJoRGPhLiXj1h+*T|mw@@FBxyYR(hqSIk+AXXi;qJp|LG--XR~}l zSHTMjV{F?Dl!=Jw#uyZxX<{JT$ZRVk>VN;gPaDrD#{$QVXI~o6emy*Ze-AlH0|vi} zRJN8zgT}L`*KgwWVI;I1?LotD6;Kw(({$bgA5v)EBOFoK5MC79UTZ)_f+Ien;vuh4 zmWm!rqer+xPjv4tutXPR51GD1gM`HobU^L;3@G<;5}1|xD8|&R(box+T63N;U634Q z#wvZlgKAf)9n?oLoOX#WFrEuCozEE0hpk6BiNqsXnRY1WR-bPkh->IT%?osr;C`R_ zm51oIHu@?LQRN}3JVfg6vY4v&)-^mtFDnmGpqtkw1-%CU=lsN<`sXVC&vxs1f&X)(y|q>Gf8NDU#sB$G`9CWU*z!GK6}4yO z0K3Igg||!ZxsIeo`|4wz2H`Z9#n((yP!W9Q8eAQi`zCx7>)#Sca57c|pB2Gp1!gLO z&x+u)BKWKbJ}VIWpz&XYQ@<^o`Y;asRRo_E0K7$SmW2WL#qn9vfQsg`qWQdk{82g4 zDsWPP6Z?0?--#-0Qt@{#N_Y7+aCUyxaH*o`%s03KOWz_`sxVPSu~}iF2absz55?y7 z4^rV^*$3VpI+Nyg`wG}IANcvQiZHV}Tvov;Gn^Hist7MDJoRw!)arv`ynKIzShTu^ z&WJBqvf5f(Lw`l950UtCvZXgYIIqa{ZjSt}7}2}u>bl9G-YE;h^2c(0piQ6R_ena*83p9 zyoF^tdqfwYzPxHgAB8@>ODd>a7|dB}v?9O!w!A$R`DNwp`Kr7<6?tXl?WxEs?-8-w zDm4*yORI~~(Q~1N3XgsO0i;%6Pp9;LB`L4JTeQBR^ zABX=UI5`$L5${+8eilVqTFPWvU@r~n89B)NwjKbZsWUtEYBfu5sxG)e{GX!V4}Xrr}uGgsTjExX#LdDNEfJA3kE zi23jr0#+yM{hRor)871{aF%c`7gasq{fI{BSWUJ1T%M~%d}dRc-Cm^Gi!@_m(q82f z%VC!LBWPFBtD7QEfB4Wsb zb>y?aLIEzw8R_Dfb`$cSlyIRDKsvfG8NYw^vCh+;T=%_=alZ!zZ#?b{(P%aulKE9+nbB( z*UgP<=+`SE0_0Ha@Dm;OF%f*cp!OZofb5N8Q4+-h@BS}ISf#WprTu7>w(;20U>!~M z+Gj?&I?1C=b3W;g=K`0ln+5ehRR;J&GRzOsQI8~u^^x)2Ys0*X512zCZ!8uba7(aY zquTAie^l_pqM%^w`87a6Wph{R`&(DvGZZ(d(C=S;tZ%ehAM5McU+o*+Ver5;_Zv(@ z5+HZ;y;9=ytIw^CsowNERN`A3mFN84mH5J*^XJ#`oL@opzg)|{((#p!f7m*Hc2M2v zOWutW_Icdpf+b{P|9#;oy_p3r=On;Gf(gI8%onKz-zgM*fBEqgXl~(EIN;iKUx|wjVyZ=rxs76dkJ$sC?1$n5REXCw>cv53}Wd zgSLODDB-@Q7_oNyT0vsf8L&!uf6dB!w$cWBxLYEOAC=|?*EL6dmfWtw{%XwKQZBUC z^qA?X=d0AFfkaMsqqH|#Ticb=o~pFXR4c#O|G}Ew8`p^@+o;U$N^?J2%^gs|5;Io5 z{C7E;9S1@8ob)*JPl@R2B*;UnrKuA#VrQfz1540q3q3>g*Ks2s`jr3zM}x`5b{E&O z<>17QFX$OTyDW-`U&WcLpR09t`-fX2Xj=q2Y_x7|+rEMh_XQm`uM+^b`MiP-6?FK9 zp@X_uZ3ZInGZpvylWte0?;4GMlbh8Zi>5zHA9GSNdLe=Y8n=t!rHf zQ$^2J`o7Zl(`@-~UD@vhL8a=isp{J=9)hZGw{MMz{)Vc)rnA<$KO`fZ3Bb)4>aiHB zDH-786q;EVc#zH8x`5R!Xf&{O)d`ks$KsSjjsQ#an zB;)CTCn0?)O!9N0o{$0gTo842eoEsHNq9Vt{F26*m0NYsx4yU3pX$v{oj^(?ZoPG4?uND^2Tkx$TT(n|*eIj114 zR!5?r^gp!l3q23d>|BK|sh<;adMq$VwR*jF`Q112lc)~#Ml6z=m_~zUpN51t^^0cA z0{e#-v%vFNBruIg!W;kQESjT^R;$(8-r9nHTdh{%->vpWtMy%bYx~7UyWMKHw!dp_ zY`%E${5#aTgYW+?+n1i&|GSg0ow#++F61R6q{W-rz6f+SeSk+rwm zFE_TIZ?v~IwUP{Jbn2+Y7)K<`kjgYWI{O-RN&{Bp<)!?rmCYAFya=98;;yAz+&PTz zSjr>}c3IS?gM3Q^(#L5icu|4-DHot0E-p(|FU#3G(@gc&4NUbH1ylXy6_)ySBI)ms zA4WsJ31xoY#`q!@^^33FqP82~=|~Zb+g085SO)z{p*kQbtwv=^{@W-WBrN!>tlUAW&u2lS@5yGj&gZF)-OwJ2WVE~XOg=pqOG_>I7iHMMxrkfC-&@){yH0n~>t* zR}B}iP&}Y1Y&u2rN-XXeUWSNyXq{r$^+LU}cT0lz-dFbBwdlqJw%FQ!^FTZT2dbmd zNrL-*>R0Z7TXF})aperCoB@?H!2Df$|L8S*0ry8NSNQ_IGG73AZ13NQxus@2WwKxYLy?cZe~HP->1_s5A?fzC$;It$RI&%5Ud zl_=sGSmG{Yb$^qsypyvLecnw74jz1u91%RPbMK*Gtk;xR0ILF6Uq8UI{0|wyfGuHy z6GBlM-w`LoZPGqm#q2OQgG0?ey&W!x8Eg&flRb&U@K@a{g(ZGn>WiXWxq|V+s+)RQ zZ$P14C0ABY(;n%@Ea=jx&n_<4k7}snAq_>$f<*@*gARG*aJi0R7CgRTfq&=XGa1fJXHs)N-(n?K7uKI=i@n}% zIe%=rf3!#0@6Vmz+wLEmE%%KB1~#1E+nW3!|D{*3tu)ag3()GWWdd2tQLU7-g?Q~E z^UBNGW$nKCFTVAkh529RC4F&zX7j&nzId_myukm`-hRGW@xR=~PsRW8*!f>7hh;_i zQaLQYGKb}TQ@$*PTC7rWq*PF)f-3s&iWa59D))_5 z?vNJc-gr=!a^UIl5uMy0v&nLfF+E!9l6&VWSrR$PBO@bO4*SSsV;fl#y~yLG6S;r> zkYy2uR0JWFEnE?VRF-gM30IsTs4|2fmLa?}#)o?%0eMVD@Ll)y(ks3*1Nb4}!RlCP zb*!|)g6c(u1uLqK3JX4VEO_Ts9XI0Uc${1uHzD76;J~ON+o({{H-U;OvW<#tqaxd= z$Tljn4OPWAMYb`OU*o2v8dvf^oU`OKWY{5rFi&Ltd>%T8+58V%ThF&!1^$Pv7wvY% z|8N&S75~Fy=YP1kh*?na*4hUZ4TPi7D`(i_afYp4g9l=|lgO5F1w*UEi)-i^ViEj` zrUA`j`N&TQ79{BQ#@$mI1rmwMe+#c<7WvA6hL82jrEp82RW0|Hz>Yhu<4b8Kmeb=m zVUw8Jg0Eqa6(_}&qq{&{^0d({BZ9c6!M&pLuVGy6T^Tcnb%AK->BBlfYjIDbYS}Bk z>Vf^hNpj`5R%{qqO<&WVLB)pg;I|FdD>L!|n2{AJMrB6cr5QN~E5_WYXi>_G>2T+< zVVgy4%=?5j6?ewckxj*&Q30C@*i;l66|#9?$Yx0#8uR&jmQ0`VHTZibE+$^87k*ik z9m^Ts<<|N*#&_~|ANX~Dk zrweQ=S9bLRyeo?-sXG;7w^REg8}WI?SVi;lZMkhKnwQFHQ#ox=#r9J9Y#yM`X0Bte z%Qf|uQ03!*M)&Nmxx=AdW}S~?Y;WMJc?h7lI<;M$+O7bvdQk!1iqfV6yk868y&0#? z*8oj`Ea$cl=`6;Z^EGtq5y4kGHy;YltEhP@koQf1yo#EqqUO0PY99Edn-HFcVj3%q zxvMbKXoG~MaW|oZp;)!aCiK^Y)>=b8OCzy*(S<*|U8w>y{o8os|5fcAn3wlNhQCw}u!rREzBUDY zj`i#2+X%Bdc`?4PZ=B%k+5@T>zHWY;Xx2zBkzVU-CiA-f)mbxc%v$iq;$`r7ebdBU z*B-hc*>bDm;QF>4GZhC{ z<(8@3GN|IimV(C1Xe zw^i|N-S@<+__iK1-X^;{)oQn&Zx;B!wp-7iSNvah@w4)Mvqz(55A$KI#)&{4QU83d)mBi% z0^*U+F-rtVYEswuB1CE`pa7E*iy(1`a2&tN3f)cq1Yb>hr5>Z^#r$L(+6F{NLDU7x*9BFSeg=SMvWZejMt@P7NWP2-?TK7^XeB z=0oa}h?5T5iLpN<-bSNUmMA*q9rOiIo`(kP#jCW8-qx7@$ROQ()fPEkHR{W1%$vg8 zq9Z&Y&E8g@1a17n+xB1fy{#X%o_oC?eh9qQ58I@VH_2AJyr-m8BhFmNfbaCBX+l2AVJ);KLs6(6Xw=J z!+!{ta6zJT914;&0%DWrUewexGeMNg7P?}Nt0Gq&^6onq6Hl8k>!F zy8;OJ@N>2PS6gZ~0RS`ge|uy5#rAf={%f^2UsU@4E`A0f>*27Y6p08)2efO<`NX8vhuh(h_t)LTzh9r!U&qgs8v`0fK#^{`iA!1@k5~L;= zIv-Mhh*D0FYD5Es{b3r0$HY&F;0-8lCk)v+A{4UX2;n5bW5oLCLW!-|X1GKn9CNgK z`jUIF-M6-mSUwLIa*A`bI${ZtEgZ23fs|4iRF%6pLOp^68O2gS2I^_nRVxUgVq?v7 zgIcm?wWq$>fHy2zsQ9vi^F>bNMZf=1t0g_`!#IOy)b7@W63BHWlf^8k&C}wnkw3G9{>dVN!$TJAq=IoD zha^F)Cv7EYaMoBPL0D}Cs*veuG?0TjA>&6hI&%61p%MO^J$PP2NJZuC9)9Sc`gW^s zpD3a$PzDI~$Pk}VxhgL%RfiCQJ{dn1`5cFsM)I2j%99ifj$RrJh-mPb_@AKhQc%mR z3F&86Lf$hKFrPOC=BK=wq><-9%%%^)v}r=TIOsKNF$-SXs(ys3plx!VJ7!V6E}x8O zv~z}ODEmhC!U}Z5hX}h5r+!t|>F_ZPE)p!rV7%B@`7H8DEVy$GhtDFBurMSEZ)TM= zS4;)9?BTI~B*$36?;{qF+TO%&X9?@I8fa+QM+u1;m%w{$4fW=TCJ9SOU@Ueo^JJ_b8*dBZ!d$Td zHfsp+A=XPahQQ;H`r{7T?;kL6m=G?- zoRZS-Vdda393wfaX@8u9SEUF}E)e(&jZzMhZ-k>1%Fw&CQpn48TAi))%{NZY8fSrL z_mZ3PQ#@#q7bo-#3*sdjtTt8or7CYl_ks(a49*&jHQfD$jMaNN$hDdvT)au}h{$DH zDp76Ely)vcocL)%#dw!Rf_&ENCrP6ne!!w5#)K1I2ai8+lE@K+RkVUoNJlh-iO7#r zcsnu=Mp`hpH(tJ@>S03ulM+6ErrHV$B92UdgMGHb|JZu{^Z%_N|%%(quYJ zSf3h0OiS%FlG0FHTyh)~3|8w0{BA>H{5N`hMmVmi<+W@6w%`HnN$i>>;lZAGbZa+ z!tqSY1rnXTN!Un{}0`O2HJ_MpwMBxJ*_ctd*!zjhCH z-tA?w)Xd6X$oPo#?Q`RSwUZPqq3rD`8F!SF>zgiInFF_9@wcO6RO#x z(zr7zX2Bs##6iy0J7ApC*m(|r%c5RqHUrL8jZ5m?`y5Nu;R+#R?+6B_j_(uBNr3tZ z8zGhoP6L%*BP+D}mf_$>9OB3)iOIbNXpJR0KcYeva8AWgLk62)TjKkNS!Q%hM9%n< zBMRFR%G64PY#wMOL`5DMmIIEIlf)aOG$2h~rb2!CkDTV$ibl;M+d65LJ72ngsAs+V z5LR!=bG4gxgrk9VASf*TvSsbBLl<1jN##D?YQJow6BvW2szUGJ$;lSx0Uq}g9Fr4evY9*yfg))% zFw<3i!qLQYHA#)Q`uz{1qCu|6xvLXq>iLw6{gC0)8putv4CF$Pf4h?&p4L{7Jqs%( zqq50|jEkL+Vota*L5%#$jOeFaAt5KR0~To9LG^ZHqp{i8GLTfJgg(!NzK80gG1Mm~ zfwTalK|(k$wG~#-8J`>rY{CJLCFnY2~$(-WI_>g5Bp%Mn`nT% z>{6>w?QO_EPNWv09I+@Ig8(>vX@=;PPJaSI|1cqa`dMa_1J4cGhxO-?&v+Dvq!BV7 zhe}!~R4((w@{JSly31e58A--aO5Xa?f{}8$*86FQ&W9vI`rZI7-fdr@9+pDiD6WR_ zVM`qAPHFwY&-CQ&VgHmwK{l6l0a~8(&`U@_6BU?|OQF_Z%88dXLc0-?tm9=&SmHV3{p#cr*&MKd7{f!`>5zynkpPz zQL;~(D_B<#&Bc+Ro_Qi0Y<_q8EM_4ajL~r{?N+qQA}$j7z5JREW2wF5|2(0K_Ba$R z^@S-~cby6&9&xta z9c^Z*-tz#LXR>oda0I3R08E5~K!r9ToQffdgx*ZhI9aB1eI&I)xqACx;^~)K&bjb; zT(kSU9rT|Rk7a)h@TkYaAAV>WeB(jVj_gY=AO}tcdeM8x1BXLEl2;00n#X=fNhCZP zyed3V^2SK5AR zHQ>LFv^q?>RLAmpX%d=r?Tr_ZvfTl*#o#$3+47y{os(Y8$Zb2knRx>f;a3)>BXE&p zVekVOJT#h^w~*$Xh9QcWa!%_VIl;*LORJkfU(Y~w90#{Q%CJCopkjR%GENAhdc_MVLpxqGD5ns&M zwdD%k47C`Y>as1S_{=W{wOp_S4~T>H0_#*!2L3x15DSxz2o8Y8;(Y}6BBy^6sp5W8 z-s?#W*ryQ=>7UxeEQ`p0RD9;k^&*tgQccnt>uqtPJneZ3HLwIWHYAol{MEWir|p(o zza`l2QtRRoft_n7yyEcJ-D&xW4T!*W)NxT^<}$?KNCYV)OR^~L%y->A#*B4m1m!Tt z_0X&=A|byNtT6fMWZ^(NpauwxR~qRZ;M$bnA*teq1dZ@# zI!Z?fuBJjP(uf6(NI=sOGR|3a4%SZ3=9L36q7mc;FB$3)aZZS`-j3;?%gazUPCnAk&x__K;$!th)4S?lEOufseQj@zHH{?l1EbU%9)OZ=J!% z7V5(Aslig|kLDI&J{!d;YyfJ^-83ItbT{A4R@z(}e%YdNcB@*8d#uY{;W!97_Kd2B zc7J~V+W~A;=xuNCNg+QWfrLOfNqYc<$DS8ax0Yb)yC zL(KU(OM(vKB4N=W)_=>oi;-x$SQB*v`C*PXRPmj>lvOT6nWbF7nsRgd4YIgj$XK`1 z(<{id%6a27bOpU8eHy6W3^1z^4cf8yfwa_y*x|k4Az?D};McaC1$!+QbFa zlQe4ZVO^s206r#gHvgV-^qTpnB$3mrwqm}s0~of>hjsN!Qx^hV;=~510s*SZFEqWR z{1idy=MOO-QlBMplSF6IGD*Xdj*wY@9NYMYh{-xqbp{V7L~nx`i0q-d?hoBw^@rK6 z(+?Zj-!-dmNpwaNri{choKPuBU9i|AFhvlHN^s)JiYy3RH~`P>3{V1FZDt6DcI62E zhJ`gW3>0naYyEDH?)gcE(eN7C}E+R;^Ua0`fsF%D6C5{uB%dwC4d+W{02GY zjb~^zVnVrq?OVB&m6sh=b%7vQu7R5heAubWjRt6!HQ17Q@(p{0@C z!a0VR+P(jzQbzb8PPqY}5=*SCAP&ou20kLEJZlODI6|a$@3VxURY-@G&a)yw)&_0G zA!qASA%)l{NLnh#hdaA_pcG2H>0YkS%R=xnx40|i_(C+~mR zJLtaIf4kQ~%`==xiPhA?q&XSKnjfX6fXLFUd3!(Yzdz_8J1g`@Ja3>hl0uFs04 z=2^SZ{-L=-qO&ucD1_&owLM?QT^<3gB_R9^1WTI%VXo(6HZu`wG#chtW0?Q#l#Ksw z9_qLx2nooZA@nz8qG%FL@QnRy=iS@C*X}-u;6!f(f|+B0m6=%bdDelBWK?G~bQh@- zx!Kbgi795wh*P5mJ|7fc_~>Ixe3E{=k(NT`3xn=7DpVZ%+t zq1R6%85ABPYhNARXr)MzjyMl)D9Qc1@*7 zDV0Et+N>@0{62>%%+vQKz^O|XMbdtAPA_mzCj>oas`PA?HRa*a5h`!hF?>4{oxK`2 zC<~&i)ZzgsQZu$Jhm8-NL%mbOt_4HmI$_);$fpozsTUfM4o!)!G`63P4RsvnnP+UG zkV4RBA+A*UXW^+7{Mz(Ft6)=~ON_i$0*JxAA?*-p{)i;a0G>oi0LRvHABWGmCxMo* zCzSGEwXqueGrLO2a5c>56- zg_agllFunu>-PuyswUJ?EEQV?Js1Eeg%tvVo*vZ6Zg1a z3GrwojYn0Z+v`q`8}Hp@fBKx-uMZM>Mv_AD4WHA-=}Wo1O}f%?(HlZLLL{`8id?U& z0uQxCY~ud#>6tzI!DHWq6#*O5VcH8RAF3q<{?GEC6(k!1en~2k5cuwtRc`FyIH8eG zV_5A0nTVY*Es(A86B6)cR5DvIsh*lCm~u(9d{G-)Q{)Sz(7RS%t7Hw1BGFW@Tho_U zX=&}5rM0gvEib;x#9vn2MzQaHD(gG$_Q;%_k>7MO`*{YGGOpTc=Z}uzNqN## zOyf=aO8KXi?7gBB+~qg-`kS%R~>#o==u2BOOQR=aZ8|vtLjPO+iqOwFG0v#j(IyloI(&{H)c+8*2g| z4l!3=Gj;g`2Y=8xhT|MuDDMyE;JV`w=UmeXoe2`qK|=kZ*Gqf7kf?yA7)K=3?@f>R zA@fhOH!J90+V=!{QHp1E$Y0ff%n1PW;ppwUY(&VSfp+pKGw2M5G(dmS7=ij;SEb5M zb7LGM$HlLU#(iO}ff4mt$f5>%OHWmpYe%=cgmMNBB} zn$RmZ*Ott2nQu!5)@GKS(x5CY38+w~c-20eek?~*ZR4(FsK>m@Xxx?6bfJ`|PL4hc11auD<8GfOTy)g~Cq)*W zzTM)^b7*!Rw0dg%N8>IHy6pn|1m0PvBALICM1ohUju!0{UIk{iKL?6D1Q&x$CeJUp z53*HHy>_ehv?QGbOW|bMxd!p__p${#7EhmNC`p1qQ73pbwGdO(=8X4h84kqMBxXZn zRwbGjDhj5i0k*N5e@WSRsGo!csZZu;Wr+Fok5x&av88FUv7c6vDxupYpW~$4osuY> zIdQ=9N(k2r_0_AQ#if4C0(!^YGsK0T5&kS?7u-X|J7{ZjV@kpq6W)B(2*c_B{PNL< z0zc>lY9CA(i4h4wtZWo#W5i?&j|H%#>==obPt#{fQBKI8GGY_^!`TR2=NXNOTTs2{VrkFB0i%fn!6gjiKMC97-fw&;?^sTGO zM6@;bkc!C(&WR9kpr>RG$jLTPItaok{`t>;mZak3j3f{V?I%=F zeJY}mNT242r9#wP5M`;@C}t6fM8{Z3C5Z;=a5s))4=yDuYf?HI;bg41EhlI8#Rd9w zae?#&0G8>YF(RDf0jZ+~y1e{^E-wqJJv$hDa|N!CCM$=b=MQld!STXe1=4~E#!Mhx~f+ubBXhpMiSqR0FL6-&=bs|N2<-AmNzQZ&^?~_3zLxtU7 z5*E6XEyHLk_Et9YTFalT zM-?1`A??`%I`&8nE{C zX?~LEz#;evD3?h4wd{81koE)+cgNVWLD(UVGoR^{E($C`!tBvhA*?Kc9h8IjCwm=q zqHNc!FtDLvT)?nZuJ(*;1Woq+#Kwnfa!!oAQ&TL>WnIcTG{eToL^>o!Z?|M4p7{#Q z3NiYGjfiKHW#U-cHF-hy3G8aC zK}3X{I~$I_MlBBanFEtt>qxzZpyaY9=GcIp1>ejfSet@y;kFbh$1}{`(&D`y=45+o z@mjpz_SWp05bAAjNv#fA%JEhen=qgv#Jy`YL1oOhLOnaV>Z_?xm-Jod!%g0N#XW)( zl3Z0W{xeO;H7bRCg$gmJey+ckOt2DH&zjGk)>J5n-iw5TmY;z?DnxcyF}FG! z5TRPOI>BL;WioF7IKr0bPL|DC`o(=#O>@~Bm$ZZ<2IrU@Le_O1M1Xos3>%r)lTCoI zbKPqlD|EoIp*)J}++uzW!!ae*UsgKg#E!e$s#T5A&^l1#CmTT;Cse|x{E^j6VF1T44i4k;aE^#x5^(9|Fqj@NjC+qHg1JP$ObCO2)6Zn z7IDRnsSEB#hJG#2kghu*B8qM*Ia3BNLh6K92h}B(?W0vWlN;*rb;F#Z)@KPOp2Jbs zHFSA-aZ&ybeNje*NSJH#Pm>VzywtgJ6(#JPYaOh(9n9fjb@l*YC9mnt zKE^~>cCq}sROK^2V^LNI_?kEsL(K-!Tu~op4&l)#BYFcS%V9&k>F5fJhiMd)lU>o} zbP#o=HRHbGK8xkNNjXV6(De0$;Gn~jfytbTVRssje#iziYBFfC{2xo`pQdQgX7l9) zk8D@(@EY`)&x&K(aI&iid{vLMpee8hWnW8zE=vYDqJLWJ9bT#uHHoK;)hvr4ju zta7uU*~Af^oaGC4*F1D^OK6MJ&tK21HkvVuOT(mq+HM};U6>#G7_5TjS9J{rR) z>D?d`)Br3@uh919wtkRu(&aof#cHEQuBNUCIeO)c2|}EQU4BaA?iopFe_VLmqfyWe zB3XG#gIE8>*ID{XN*BH!_^(D|mJH-CijXC@IF8?)lq`SI%N`CGksWU6ld($Tv#AU-z7t$h}T#25LPLlbRg4sI$3tJ(;bGnQ+v z8*sJ-YO`2A8X+2)!w^K~7W9m0w**%JAHq{O!UBZ^b72?TbIT>A10ml*!*?mj9M8Bg-k+g3GNilH}@?Vvp0A@G_A_lbeDUFd7 zFCa$@G60I2v_~XjfkQNWvLvg8)e`>@(f_364GoDRfU;j!U|QB3X*u>}!KVb zX%s1vDuqg<&^2Z4i>F+%Y%$0FG|Bu zm;R23Aqy-)9zy{UQX6Hi6E*301N|mP?YEti-Jd7BDqslfcSbg@t6@HVySI0Us772N z1s}5O$bpWJ5Dy87p|7`FQk=0U;HnECP(`(~qv|l+1NNis+*-HCTM}>sY5ihcsP@r{ zD)kuPmu2eHO?L?wr2&jNFo;oj;fDlA;TZX$Vfoa{P6|mAg@x@RbSVh8Tb%g+jYg?I zL_<1|`czv}Aw4B%bGv1~9Idv8Yp^)_1gk-pT@$g$(?P&5k7s;{EK0q06YYWtXcGZO zGE_J)m(`nf3xecV!x1=5f5E_)nJu0%OVJRY5rmNr-bXAIctAi)!M>y%>ln*?df7(| z9UY%E-|Qc(8&XcqJTzqIDj*K3R1HJtmPsumyvq*3*1{wj$qu^R}{Yez&Gf|ZJ0u7^(E?MUhs#7 z5sr5avR?!^IG-~IdT>Ks7C{h&TJ8--DD%qr5MrrLWs~(0v?0^C-e^OQxE2vwO(VEd z=aUv#Z=fuxtHFrnL@1GEc9xrB6ubMUnMXIn$D!GdS+{!E@FyUi_H=?Y3UtKqC-vl$ zVh>1#+dlPC32+<;g4GaBv^57QEYRR8#CidkhPAGk z03l;VjpTmOeKR?g&{pAqqFd7THnQ>*I?GZ&!GG zCf>FY0r@L+K$V8_bIiu5CPB)>rfh>baJM>hCuM9@k=D#o`h$x~C)dX&m1r`s#}>ts z0fm!8j#E41h|M8#m0mFWbPnNorHsPI`ipC1IAnbdB?nev7E z0LT3=d&xEQ%K7&ENpSp42u%_ zl?UsQ%u4*%Nf-_V5IQ1xpOb$wrAscN33nL*lUJ*aMr z@bgOhI#|LehM71*HsjEoj&Q$W1{Bc!vj_}uPH)~E2-w0N84|GVo(Buq)4YqJfqWg0 z01uc3SBRiO1QjBv5P?Mm?ltZ=zyvPk!i|xE6;?~Z2DY@jMh2OZxDp#2WW*G6qJxRA zx*8wAg(UjMUxk`7>Uk?}fhNQJVN%>LSs}_L=y3Kt0IbJt2BeKdCRVhLo3L8e?SVc!=omnfOp&Jg8U zrE940K!wujvxc)u$(<)EIbmd^;q)?`@tG*izB?{ExU_`Z*5C{i`u*w7+qX~EM*2Lb zp$wnx5o6+w-Sf{*{!V-@TyN}NwjHxzXX2*JS-tc#;{xe>?)A;(#ar*EXO0ufWbY}S zE?K>P;9U9%F&)m0-Lq9#rnM%IM3=P1(oVBP>3JnoS)wP}Qz;If&XE_+0QSaUfn+yF zMS=6Rw&)Ie*?QS_wA*3+A;18IZE!4SkzhUx_1*m1b}4U5BYzj9KCB41oN__of-zN? zy^Ono_C9MsTbEWX=FYmayH4&Q^Ou=VicQ)`w5OoX5h7~%^}MM@-jD~hGdE= zV1C0-0^SJ7ncHzCJqFs!VcKnGUC>bd3_0_I*gR|h6rj4PJge+Df2Bk& z{BA-xCjv1WGOwibp_NUxv5_@A3I%AKE;|bvdrC-L%UHA~ED&VOe2pmc(iWow_br%n zV0F=`$I>W3BEkJW_2ml)mUHO>oD0y^q=Bsaiuxrdp|7fqrQ*8P?djTbE?;|xyLSBA zn_D0X5InbqZe1mVhC?-GrI@zK|YTa;&eDHml~YPhCr+l?B9H77cr& z5)g1s2a#4;bKqj<-rR@<#Iw2unEY$%A`0`Z&UMo#EF>y~_4;+&J96^u2NIkK4@ZF~ zY$f&z-|Ug^hRkYXdDvH z)|6e2ypmO~fx6bKS>k%FHX&qIR?t7-2tr=|SQaTQ5@2yt7D-8$7Y=seG}H|vdVN_D zK{@Kv5X?^PXFYG!iq~FSDv&9y0aUeTXx3h{=&3MQnHX=T+8NQubhO7(-#vXYBuKXw z$hz&-><#oA_{SB)fW8A91ez3H8NZO$Z>qFhePHo|pVoM3vnyjANtg)ZYtxef>Q%^X z;Bpcv<~#QjR1`6e#>j?>uB@QdV%tzaxS)|5Bw68_?PXe!^wT<7r_0)=$A@R)U9*m? zC`YAAZazFVo)NWP+!4F7YVuiXODeL%Sw?1|9rtQ4fGzeFxM)?zaI9gqvq7VJPHzh7 z+31^qc1=%JGbnmS3n)h;%HgUUmYB(Maj9pDo(_jB&`uBx$I%pubwBc{vzb-17iU10 zk-AqH4*&QG6(pfp?VGX4_*P(&0c{gwns8+uP7);q>s_xw#sjmmf__W|FpNr*m#?F9 zqOKWn5*iQ%GuG^WsqfL9r<#;?>ALzdxRFt=m2;9i%%m!w7$iM1#WQlih)A-{%&EO) z&LO}~9o*e*2OR*Qo#}8Tv}RF+zow6>b=5%Mzs(53HQOnJ+a?m=XbN&^_bh3HL{8}!3v>&bOji?U#-4yr7r4@kF+o#R{m0ABIYGE4_u6$|bSpi{j>p#|kcBpI> z8&94o*cyz! zlHqhy*{u7*u|(B|h0U5JmRa={wj3!>`#t`q{48Lv{M{6>*Z60{{O(x?N%6_n*5(4A zD3}KJBo74a&N}s7Wu@&{R{SzI$rT?+;~(YFbkN^+_g^2Cew<)-m1sXM46G{+%u>GL zBG;4Wj~T1CW-TpgpPu6+02+KOD7YfW5@b2+-&w=ZPcjUXm6r`5iXICRSlDu zOlod!ef^}=q(AB^cikGzL{2$@3mW=qcuYh_{=I7sXb0m6kEoAEIHG-N?VvM?&BvH5ju2e?~j1(cF6E7lRf!!-2pJN=s9H`CD z!ODq9W3wCR7((XBiYeZ+#Y6jdv`t3ONH|`X-3HgJQ4ah3>KI@CmS|y+#r&PPUR$xZ z`RhG7`S$Y{-?g{4Uu?A7t#)hsyVgc~^TqaesI_1{*ZN7hz{z*5 z8`o9*xJUBSZlIrtP$4`|9TH~{Iv+Sm(BNqKVeG#Go5oQ5+ zUIw(?D^e^AwRx*9NkPS$8+p!2^u=<0tWK2GPs31_S4WLCS(eA(vV$0l;j8*ZLyF(V zMEMR`rBir4FO2Kx^768dz94lE0P?(m_$(mkzY+0=tX8XSG{6y1xQkO$3M(6k1fb3U zDLKtBJ3R3G^3wQ)GuB}VHNlyzEKVwhG%}%9fOS>pm-VjGBk1z7F~WkI1XPlK81f zp{_CQ(WsdL0vO`I{KX{ULz_5BqlW%df5>YhFb%m1YLHzsRY}dFewCR#IZ*(5zE3ju zjy7S>@t7k&!F%lX(@Rdkub7MDOyQ6feSABOi_+0``n@DjZRwEqOg&`3v(QhOSf7)zH4#uIH$J zzyfl}5>cO6SU?zHnkRK|BGAG6*L&T=_eUqMo~*i3Yf*7k5HS)KDl6>w)*p>XH|esT z?LAz*O4lYEx_?+`rYxD?w6Ro>5ww#FEK(^C_0380nf~LlZkpo{nHldVx>21cwj0fm zz;NfrXUOcqERZG2z|Few0~$U+cQihjbumd}D+C3f}?>u9d^dHDV{L@E`!2mB}tk57o| zIF1|nT@jh;)wJv?=lWzW5YCYgIU7d=RGYcSv_5r#8QT)dwoA%<*rMM9q$PcpfNB54 z^c31uw>M4*VD)Qlvw_~S0gbYK2Mgl>S8zdM)Ye{OILMGFt8)q#tktv#mgb>JAk9k3 z1&Ms(0)&-i|L77Cg5x9m2S0VtVOBMSawwB2`2PFvK@e!NElw120v96e>LOGq(f+u) zDrPF&nIt0`L1ZDC@#B`b%&&IFbLseh5&N&JU`a#7;X-xHw*T6jt?gF9{@Z9he_q*t zcky#^(R@}zKk4{a?T3I1ndDljp=V7kP?RSr6HhATYbL1D|JCAaRy~RQ6&DI zSEvEejrMuMn1IK+XiteRD{u5M#LLUJ%w~=bG5>{(O`f`P(A3vdGSlPgW=NS$ ze%-2I|C08(&hrIDa&#ksHrDjooSYT&w>3K|bilUMwJ13>%a>>&qf2Gy1PRx%m9 zF?B2I%|yPR=`ne~%1tML8OMnlIFhdeb{Y{^yI4E-#xVfL^|JTVS5c zuFdkHlN~MUN((RaoSn_kq((lzN05V>9q7;5GMh8+Vnisc!oT$`90#z zdX}!3Y^2<5m$2Vp zaDUd!xmDYQ+?WvF**;$|Ic$HIou=~Z8LlUV{wqHjX5CURGdU~Aoi-WDf!mnt`h(`N z$_l@F3isrU`;47e7Jdy|zaT&|?ST89>zgRkzUvp(MMAOtIi z%rr2B&@3732RSe0tjvnYx=csUlY1(;a#o@xte&$$nj<^=o(V0aPM6MET+WxCBe90= zpR<%aX|^q{NMAi?Y58nfacW0+U!SQvR{SY*ms*mzWi@H#)3LH<221ur<2G|D zn3IaLOSBVU!nswFc;fOAq{X2lJ|@@xDY_8J|76Og}wCb(~5}h@Edec4L+x>8~ zfAX*H>;2=MAK&hE_Yd|@_IKWPcV55SKj`ir?d_cGzdu;ZXD$oiSesqGD(`UT`1rT? zN3XlxH~Vk*7RmzXUIlN!b=T$F7hh0sdy7Oq`hxzGGC>?0KbP19+hv8CO_RxjZQ5yt z8-P+$(1gP@3p^>hFblsHq?fu(IUh1}d0Dep2zQL%DwjbkfG5+NJ~^-M+h1M%Eca)j z{U0ZSbdfq{@BeIUwzjv5`#&4cUsU@)ck#1=j?@guDGbFk(bamVn~(wJA{pnDh7l!$ zko9ob@UajfX{el<4dY1Xv)ST=Mxu}E|HYg2YZYL`P)^mYpx=P6hNI&&j#(l!@-Rx^trJq%N&3a8HQ4#U#v2(uEa4<;Qj%NqI%xOq{Y@Bi8mRJK| zo$FwD3+F_g1Fc;3uZ|C|>kgr;$+)xx&NR~^NkF(LjMZo^oYQP zSPI5Fs&i_t!Bm@>sBF~1<9$X)9dO9@X-Fgrel?N{+8>gRF=v|3YI~n!6X4(;&j&%* zs7*C{R7i-Iy@g5B*oABW!|av@9$Ee{^aY6|jeNAd3BS_OaoX?GOzCmqwxXSM7c zgt^Wx4Z|^#L#Hl6$8ikpZq$Aws%%IvWDVdbLOtT+loL3b=N@fT@uQT4`o2Ioer{$d z(!^Yy{vvWg5nDhTVS7{`d{m)-mPDE2^AN( zJSbYDq#Pmod;h2Yc#Zg1tJhidLsQh|*|EYkcT(;KhetCkN*IBM{r zvKq>*S}ho~Njv%8K##Y&*oGr;mo( z1jim#kLZ5W+ubuf18NZMDfz30vDoikY2$4;=!}rrl58eqHvh#3lTvD-nU~{5UG_Z8 z7hcWEtLK)YFg0B^@31Xz-W9oAZ#)g(GNw+%!VS<*fjrsx)l3|XI<+L>HWl24Q^TFl z5fc&>n@Ag8i^zEva4j>cY%@?)<^nW&;fb!d^9rqojGd-2s-Ly%sIDA+^%8~a#*VLD zrI6R0iEZjKPt?}2eh7Pr^|C{96RKx5!BMai#zTCNj?h+XZC)Rv9f>0%(j+pZi3y7n z3RaKZRRVRK^sqmpGvRfXC8ukjnUEf*pdZdQ3An&}iU%{Tf;=coF{o3bh;UA7=Xz^uKF` z4lAt!6$0t5)6RTv�|1ER$nOJ_{15XoBSq;?HuDNCPxPhb%xlnx>m1Glq2-<7q>B zMJG8d;LWoQ`3VTQpE>ot?vh!lWv}G2Kxd%qc1+BI*OVtIZ1??`1_Lrvz?P(*Dk4n4 z`OG}|39w+Mc_*#n&}p8vn}?I-7ALMvJZqxbL<>z)+P5TKlp`+G;kzs43gyR;H0xjT zpAyAXCWfTVID6gCZYEbwVQwB+F5)?wZSw9HvSy#b60|z=Q?f6YA0m>JNSO!%^~PPG zMsAvjeL0_qU0bvtV=RUz<9Lz&JJeae{(gJIk`We~WArc$HQfwvp#3OLfgoiypuSZ8 zjDa;$FD9*{#)V}yD!#m2y8>qC3bGi|o~Uu-vsHN3s?Q=H3-zk7DN`&FO+0bYqZ}78 zj>-Ez^!y0NzpL=IKY&8H4@Vs@VcmKtvuJgHC_uUj+tez)P&R07O}hY#lFS4UnVs#c zLeJfi(veZSiU8+)NPv<~T^;4zfu(oSHT)f7!|0-!%VtDODRMdGmfUs9nn3k$fcm!c zO0l-sw+o&2umMfwhX;@wo$1#43dmrtb=s%&EXO^z!#%YzGe~^C0~I%Gs+*&1*9wX* z%sq4X7HFJ=Afp&QM=*7kL(f6*F~%XL=H3ESpqI;&3DxWt7@Bz`&oDvtfr>KXRM0bm zPRZC|?3g~+r=0Tha^ZGQY`MRXF^6T}d4)dJa*sPTPX|x^4z^|wb5dVNb!%$=;UwTA z5l?@q2}EQ9)pgxdGM<_QEZS~qNzyt~QR7o9gCt>qp*cV$GCkRVu|WtQrHIP9dO}Vn z1m^eo5{7uLgd`#f^}9^|HAi}OD)1c1@SkZya4=5;^gV<=8_l0OBI1g~vVNIFE}t$9 zNF=Bjcc*7aBiL5$8kr&U4skaj0p;Z*r|%qq;gvI?D;rpz)UZ`{K~Hw&;>82f;F~jM z;*>n+q!-|`sk3fMF6fSKFJv$!^v;*A@)P<3r~YC2dAXQqD@aS_Lt!oJq1lhMay`LrtiNre`Lw=_<6G^omW|5$g(ip%&Tui4M24A92n(C&( z@wixrd)qv=-y|F@R$?`a(@rRItx$%0sA4rFQLa9Tb%UU-8#S3Ve-Jq#1M>MDFrL&u zetfd}e>&an>i_AG=N|I#JnGRTZY63ngb4 zJ^3vE3M zK0D5(fb*L9(Eb?-OSkI_jD@PT6C{Qg*{Su$6^sSB(w$q_%Of54@k-L~zIY4rk z2J=p77rWKY>SHKt$j&3#YWr6ES&S!7Ch~ll%2Njt4Z<<;=gQj9i+yU0dc%#5D6L5nYd_(GzT6L7x7Nw4?&aBkjREsVzJ69xDF(m)sL%i{P`>MpMY4sYK zpboghw${sz4%7TSw>c40R#bKeuIq=(iHCf)65)%cvdB&oN!zIG0CAo{7S65047GoG zNA}koMPjb%hNZ48^kiu|Ah1OilzXUnns}Z*u~#UWg0joVuGbIhs_l1D4f3or)XLS) z2Wwe?>eScb6&qjA*&lhK??@1??$tw%y~52^dy_8zqDeDb^A%V{%vYF0`(Bn{nKH-} zZ_iQl{ki1Ow0sN6qUn`SS>w|(OqEHqn|WQiwB&X#ET5(oHz}ixoIt-wVK z1TOnTbq`#~maJzMM}V0-fB6g2g(2@VY7*gRsY_{#>HwHPXTRu=a_KPnA_yfR(4+{N=rCuYERTir_M(i?5VLV=eYWrwv)S{8d4iyHSaLw|l=%_Q}P?RN1h2ZBs|? z&HJ~n_l_pUBd70xzJ^m4(P+@L+untOj`ohmEm(Xr)iHJ~xy*3uDHqWnWd|C5q8 zG$e3LVe*qc9PIzkhdm&HIoNr(cU(Hzu3BNUrWhwU@rSunarqWNwTC;UogrnuWtyJA ze~Y)}l*T6^|CJ=PKb~yK@h|&_-IKS+-Cy^P_TT*L+OvLU6a0k6qNMKKqy58^Yg9S4 z!jPVkh;V+Gu%7L-+{0zzQ&Xc9qyIm9f8O1;ku`|J{Wm{Fy_9<_7JF&49KZHnqCT)$yuEKTET#Cd^NiXRSGci?gk5 zr8L@}r}>o4-C2;`UXHh|nX=$Vmw&pY&GMH7B(nYh$Bkc+VDdgeyC&@u9F)w z_VyFvg*Yp;C?}MTs4*XutMYWXAOo=V@IX}^u8sprdRdzPN#T8>+)vGZTLt%%;c4{! z+1J9?Y?NoK?deN$Z^^b^jCV7zg6g+4gjmjsRe+bbMsW0Ds5KroFmh_Em-vROFJ->t zpFXwn@3e`qJr-BSl|AD%FTu5W&X+-FPrX(eQC%Xwf5k)eBXsZXVdRM*0St?f0OHWg zBFaGg6%OvQSqzGwbG^Db+uCN=;M=yW6&5?=c{uLPJ@|&x-rR$i^VsE=+ak&QYs4nR z(%9yvJO|U}ru?hnyEixGYq=>;4q9yr9+~gv-PPG;O=6t7Ts;NgFVFr>S7(>MoL$~rpT9nP`>ud+D|jpGoG!?kindNfEQ+VIK3RCa z)cTyfIzM}Jz3BSn=X9eX`|E1RzGm|>Oa6@HzTI$_kqVtT6*Wl<8(qytS5vmLzK|PT ziN)G4@TvIGRhZc%v-iXmPX?r`Q9cKwqy7^UY89K5abz7d5np@>o!iz1d)~SZ84sq70ckD$!rwXoap69oT&$c(W zDsD~d)<7k<#&xS!%{5rI>~QMt^EOJlzkrf1*&-Esx<(?6bvMJ{YROQ&p2J%9 z^D3xnCVLrrvZ#t`O{>>PNwvn+o8N-0cCt5SWPfE1)n^EkQEuN=jF2&>!eBslV#=JgtXKm@&e~RN zekD%L6mL{Ge_a*M=ZckOIq0Duk8!ZD@GZquo{LbCUMgqs=MrZgsjG)yLAB91ZK&WTc~EenL8$1fX->20m4?WK<{4$pl8sd1GXjUXhBVnQCMK zl}L@kAyqUOFv(FKhv6?MqB4AfMLn_#waAK~n_ESaMpn=2MsoA*lIawtEvlb`q-ZuT zd9F0yIVyl`#xKxvigG^<2w*f$&MW`BtZ%d2;vHvBa}wJ9>PsExrm66gc>4jip>80Ff#Qm03_e4mJ{_P}xzrlAK;&@VVj+FU1WX(tf$0aCw2BWTzhkAb78D=N5O+$_aC_{mZ5|As# zDPo=T1zwaalRP@FVQ5{j z?K|{91$Lq&V#j_sfwlAIEI41aDxJC^QqRmUWm~O!pq4$h&9U`|Y~#)jXiM#Y z++N`0$L=@o1i5#-wi`zg$M0zkKqn;jJupOI0s{|mtO}`8Fo18mg^8AEz9S3F$*~!L z=4U&5_6yXo$ga#!cvcI_lDj|1CMrxeO8S1~+azte*(TBD*SbX#AiwkmSrU*Xw}*}= zo7)}w^%RC;f2?wQ6uLHd$vmZhN@$yXUnNygz(tI8?k?T+m)=~fTvXjJU)ffa{Zs@?OV;QXCo`cbFTNb^cxN zyMyIBh`o6`cy}s0Fig*MPZw`bZ_Y20I@Y`C5k|gOxR(}hcOW1p< z9_iD(h5zf>O_cZdHLl;PtSAypC+WI%+RooxT_3-Cb#w9R-OuN5uBxZY)oQxR8LOHB z>jao4%nk19r!34bfptT?zP?z^5pLA8eL+PP%X!S5#5~w5+r7(zUa}Pz*o5K=?-B(b zicnO}D*pV_&E?sf)3eL-H$Q7@5Q|06gt2hY?bh$Mj_9`&>5j_`>V@^AtWvnqOrX{r zKqzT{SGHO;f|KJ{uYNo}`B#lqQnnif*M*|T9Hb^gop)^c(bU0nmA|biy(;?or<;r8 z>z7wIr^naFmSJy?QG)!9^xFY$?>Hf36RAh9X!q6Ip9{9IjoS6e*p3HfyH+gNu<0r+ zt6IN&`^)*&`P(<=Z+>2^3qvGoa#dx}*I(e%%7;@6a1l;za4dj*R_44QAQOgF24Iyo zMp;966wLU56^`7_evwB)(|ubFx|4rjyxUSA26jZ51;CgEiSBc_K6|rlZK6S7hq56y z?!68RqhL0NjF_3wIj2Bgu4joG2^l5<^d-*KzYKylf*H+jIxk^_M)*Tk&f{;={#1!) zwKvqw25q|u?WBy?S3mSRE0*=y_N~*V&&MO=KDa)5Nyr^dho#{A;^%#l!{Odz6a<(7U5f7EuIuA z2w_~h6)NM+kHskbWzoB(m^*(r(-#Cp2Rv#4LBG3+JZacT*O*77<0+W|ZCqBf9SmN>@ zZ_MT2wgRFiSk!H-2;IpX4ZaRd#yWwte6Y_b)45JQe(Zh&0>UO-PF4ma*#)=FMqX=b zbby1`XiB1oyxt0_i!427Q{>@zIsmeC0Y_G~Oz|J+6`o>qU+Y)O7NU};HuiD?75Hf3 z6|0VEm8C?bl|+6b>TEFP9&P5Rew>?V)EQgU8Coc z&qESwEidY=F6LMjBR=~V+Ip37UhJkDjmMYx#(tr()r24}v;Sy3J|h6zI!4&YkVH>K z$~P34?Tisar6^Eng+Gdvz9yb3WV~1p3$6&W6Gr5B$$_MY$;hfnNG7@1rN7Ia6hCATt{iLNjJA`gb}5$pj=t%XB$~ z8J=%0^Q|N@VVUfZX)!5X*RDniWjvPIYErhU(Z#r;hGt7;TO`+8q4^4JT4oD2RXr8d zb?VQ*J;5h}fjdD_XG#LhNQ8rN*PXzK(Zu$Tp=szthG^IIW6Dqz5g!RF&!&C!dcEG! z!GZX<*X!l~-Rt#_{?8<_xin~zxDR^`-ew=1HI2Xbos=T!RT+jrTfYc?kjmL zqHQ#UuF|(j*96l%NPJ8#vb~+`2YGnpHn}7|x{Q59cPuT$6rCmbHYqMd#ycE%17N&u z!4NBd3P!u3oTK?EMipsnl$44>$v86Nj2hq4O+Il-3H))Uf4Lj zfFb@lB5_Cu;C;LO8`uKvHo$?0U5@Yon;_tl2!Z2^b1>!wbGJ9(ecRPaUfSN(J~y=e zn=PeLwF!D z81cWwchQvCz0mW3+>ML?%*-3?f`X4SjzCWv&T}q4P31((NiLgW@MzIaW~SDhOCu1K zeHitiXX2%NO@Nzwo1f=uu*)~K0DLjf%QnxK;XLN2|5F%RZm+BOfg1YXd$FI_{~qlh z>}}}(Ivxw1o=Y=VV(Va~%!wV`pH5M0vAvj+anQ42odMbYX+Ba6Cp0Cd!;KNiLg#<3 z+n@ULf7HvL|GmSb{^tCziJQ$iDs4({^j^m9G)oPWb!`FBc! zPE&kPd;fcwm;Vos4tksOzmDf)l|lcv?(X7`hxK;PC6kyC2ASGCpXr&4{-e+*4^tG(rTwp^{|9@A{XGBw z;>E`PYb}q)!i6|fF-2DlGBk?)E5vq+H^COKpq(ZzP!fTgoB#|%+D+Jj(`1)@6|#jz zx{#$U7{0W6xjE`8ua`sQ4zk{CnFWP0Bh-aH4#viUratVUk6ozUy7bTNovwOm(WOL4 zMoW2#DI?K?;UkdLEf$bfwg0&s?i4b2@~ZxlMb~;|S-PW=$xad66^kH4V--)4s$bR# zXOZx=t5PIs6sksG>}vjGZ;Tcgj^Q?7^Kc{HtIURM=73OOW-MDzSej3edq?A`(hk{= zTwV^A8F6Sl&G%Q~Tb%38E{tZn*P&;eYc>DQkB}l^mfx-sX4^Z}aAxMkfXeFGWrMA` zZm(*t#2My@xpb`rA`uPv@8)<8QwU~6Q6~V zho(KH_$?%!1xIgWkI((g!~YBGzRnZ|c!VgUjfH?Z{eQop|KESHx3T|R%k%N0&AaNY zG(=3*0R_J&r5@ZI&aeEL=lo9)^n65VM@Oq{hyZHNe}C^VfBp}82OIf+Ess!YQYLiv zKBf$`DWh%BHe~-cew1$GyHk<=1>O9Sx(RzL8MffU^?RNG>J4)&?p$jfC8Gu zjK}qm{!G`Enh{fG(&rZg@P-7aAZE`kgsQIiq9nHBBBFc}`+u$o{9zgsf5QZR#%C`4 zAHq2;Kjc$`|Mw5}dO7^x-#^;$e`|S`!~Yek!#qm1p-sM_(lQEnO2qw=MEDObxB3^v zJ66r3=!>X?VHoBl>++s+QOUG@vN%&E3LyT;s8Z~03IjMsUT3I+!e1gEL5dtrYD+8P zfHD}UK_}I;Z#1h(vW7`f;dC=u!$A}j{}v6x9E&~0!KF<- zZSGQ|jH(1b)K^N&s7i=M*rl1fWLrg z%<7|2I95BLZ9vnV?F|uw{Y*L`3E}Y!MR1JXG3+b9sM5Kx4av5)Y2#+FWI1%BALAQb z5fF{@gclk-xp=qWD!F0&)RxE?%okc88w4ebQ?WEdG=Vd$!wTzP12gJQRjn$f_%F<1 z&cpv@(|2?ln*&tyfBpV`uW0{ww6XtN%Ojm&^vNkr3{&HLx6&K@T$-5AB7J8EI^Xra zt?p_r`?D>t@#;Ugm;Liy?pF`+^IeKo^h@0d@?sxF*=yU(Yw6+alqcA!@1+nw!sx2-Z592wafI2! z2?-ebkc}d4epongW&;=|gtB8FLyC;$G+utrzQYI;F-#v)nw{>#r<%CawS9?SP8j=s zA(xN%bdi3^4q06fn)^7P!yuoDARE`p3Ti3Pp~=>ge$S1a%bOJ%9j(9B(OA6|zPo-o{0}v9*S@MU< zGHu$>!KUM5^4+yzjw#x2GMy&yhzA1YXabeGGbp!w>#WIBcU zp4H22M#qnZkfaS-;@dlWf-44`KRzIzL%Dchst~$JE0{O<(E}_?v21V2o^V+~XvvBU z%2GzjXTn(cb5`{Ll6(x^{M%+A+6QaoPtH!>%j@fln~S%X*SX&6GDBKs#FDkIX49!g zUQ6k_n2IwC>e(A%0r&ds`ttnb$_}rL!BTgk%Dr^TL_u5D!Yp(ZxeE;l^BD|Ncxv+) z9$62`(gRy=JhI%(=22AxW(BcMPUTe@hXks)Bv8(+^!FM-H#5@P=MRmM`@PGiVQxy~ z%;gkCY#NSe^T8LgrOBs|Gi|wX)&NmVWG04Z=1cpGjG>wQOja;cpc%bZN26W>fv|Zt|g#Nd&IY8b0|KWaK|9h~viT}Nx=VLMci(Lua zSXKChF8)`nNnXqjz$U!@*~|j07x;^>QrQK5`QB@MonO(!JeT|B&HPEO`sW)USjWZk zf*>_D1E_2MD>e$4V`xueB^NgsC~vvB`9Om<{yfG6OYnaIV)rDv>x4qnR>2C^`Trlj z$ou~v9lSW)@PF%gmgN7`9+1jb$H!C5&VzA;XyQ9$vJ9#;KT2s_cgZ`vF&kP1v)92A zjzyTh!sVk&rkt%A0C~rp?Fp!!0E@Vk?t)Q7ra-lxuu~!NuGVoYowhXJJ#C2?e`FQo z#d!EiT|b0cj_-i;WX9Eu=npy7b*HH&w@le=Q#6k(ReH1xf)LJyYMbbZp_wZ-gB&-wMgN$K zQ?y!4cJ?$78Cn+}Ls>M&$X^kOo$1*|1`68((D%tb@B%8hd_PVY-csy#xNpT+?@lk8 z7S+gY3oz*4$#CVyqEz;7;MPU3DI=@Fr5mLA#ASme`2WUUz%~5;Uayzu{}227oA|$L zd6uM_dJmN7eplm%`&HQk>lyiccEP%b&tM;1^xkaM_QQ=6B}7!9(ugcZvi z79pl-orT^dv~fYPr0zmD3@=o-#+`2ZveLS*foFDe{_5aq(xJ`sWuE2fzpA;o{%^m( zpZEVhIy~6u|JL#>Xq}c9h|}%kxFkY^Ixd-Z^WN77ZJDZR|li+~}O)lO_)7HAl(ID{BY0neXgilJr=j9h0I`gRv%de{^ zF9r=&ElbUOgv6f2!8qBL#;b)u_?-;JS}3oToYg}PyR_HkLk7xpE22LRT+&@<4x}>2 zLJMBj)N+__<`g3~Y~@$?%*Fpn#r@ndfNJ=^!`{(BKK}dOi%tBWwLEzyN-eQw^>nPC zXX?Xoh|lR2;~z^?#OgM%Drid$tJ}MZL!k*!cS5NnTwUX|q+*lKE+Cs+KZMOY(G{$X zb2e3jT*<>GH53XK3IxM3A)~Fx!!8_IO)mxoqF@kgh4lb!6~woslDL%PRj94J@ip)D zZ=*-LT8VY1z9P0t0O?dN#!jy|jK3brN;P?UdK&5ON*r0zuB3yCO8djb7**<7MRFf2 z2XCl>Yma7rEI404bdrCYWmf z*fUpLBbGwDw!&7p02o5|4&~h~3y_#ZsBqh+uxQc6tPIU$A0fLI-U1NB)1l7hW#Vu9 zZC}Bc)`09y12~Kxns#Y9+(lqIbsMjR76A^D@yFcYK zkNl^so@(uV^G}`rfA1)N|G$5@_hKXet>ekDJvnnd`>pWpVR4Z+5?{>Ju=c9b^|7$B&BfxVTcr=7k6kS8!~}k$#Q+2itT7v`rj@f? zbT%5ITbpHMM0_*=5rSULHrp_tpX-^6{~P1rgGy#M&m}+&|98~i%j^I5_j()qkF`8= zXJl8~q9Zp$7?BSTYD*ODD%8&TJgi#+76(M6Q5@)8-5jZ+=q8Maizr1@A{Q7l@(=MH zMG=W^d@{~pqppW$UCKNXGjI>1K$Z=$2Z~R32BWSE-3gMXLjK|%xjM!`po z@f1id2?FFwcCg>uJCJ>Rl65AGg-H!P5vo7LT-DPQPi3He`VUpqC*zwDiKdWAWQ_yx z{QWbAr|8BfF7$y&@c>y&e*@1VG$jn(K+lUnC;1su6!9Pbi4kRB`xyu2CPE_=p=js1 z8k68?K0N3_+%RPsA)kKlADL1dt>=pHQs3r^Q50B;9cBv+DZl}A8J?l<`yQCW4>uJ5fxhqe z#>sR{VF=_@#o;)D9sBBaKNtmtJwx~R5RKKJV?0uS zg~2!n+YCkzWyAF_6$(rMK>PL8`5AcM?;QR1f1I{4XL^&en+bv*ioi=k8F)4!lo_S? zF%$AZ7lznKEo&^CZYqRS&Xv%_cDv$)iVK7NUe8#{Pi3uB1S5B{K+8r28V_$q*k>qG zi53zXux%p}PHeW(AALJ4EC-|1sz9{4R18T17e+Uxbees2%-y}FY7y*}9Q)fR)n12~+P zl^Q>yBI27>py%l+OrHPlTnNtKL=5v;98wk`I8}{EVpP|LsM|S_9fEck z4}I*m&02FFfo|CuNVj$F>P$xJNV1jrWC(r7g^c;B%Yz5nqMLRuRBYNlc;1F#coh%d zMSgqGZa>ex5a(sq?JfWxfu}mC<_nl z?pxYb^CZSpHH!54A{8H~Em`49bfW>eu2*w$^#qVEl+(^619V|ynbq@M7Aa%~6QTPEGX(x`cHNwB^QG>St`)rPS*Ms=BAI5?g`?5DOC=9^Ln%VurE5AOn)Yd!O> zwEe}*<7@ABHk#h`K6B!K9dddu8-P0h|Nc?l|L@>nqyJscV;FvJ8`-?G40=ezLL7i- z2wnc+EKLw~0z){&K4utE=lJ4Wp?~VUgcP*pmg&y=B8Xx4f_OT(xxD{s7qWO~Yozoe zTW?Y7u{bfv(M&xPmc&5jWxplTpsi-Pr*?AxAK@AH(HMCHFoHfsEg-|wT<8yH)P=qZ zyQoWWp#w+AJB>(aE|d;%bEacF06M4Ja_dq!gKvb2k=f_gc122xeR&98<*nbZz}T9BtJcJAKb&pJH9wi z8_?Nj*)O7Zku;uvI_p{`G?;XC#=YK!g$baj3J{IHum14X*0^u>b4jPl0v zHSst-67Fa>VDNpyt$;QsT_B9e40{Mb27Cl510+DIDwq-vff0#7$_i*s1jH11VE6!j zmW$Z7&tD3H=D0pD5ZfahQTuT+VA(NuLe}l50(y`p8mn>v%O%rM8tH^3po~N!`J166 zH#?>Hk6gARg-5o1Lhj-av}gS`Xj_kT+v0F(p2r=_cjJ8TsCkjIn|N}pqhp6}S|{yz z-qWLIHRtV?e>;ap$80;JSZSo9sScm8vA+9DJoE5>5%C*Ddn`V6`k%x7y*&SSc)0QZ zSj%JK|Abb;jSyrg%lC;7F-2#J=es06+9`O2i&8LV6B4Q14+Sp~Uh%8o>^`SEhnjg3 zEvJ zl#ZNbf+1Gg8p#5^e~vyd6o|H-|E9Pk%{1vEd5t(my6g-^LtRdmjVa2tBg6Xu@VU6;fn@n3nvG*k4=63hP8q1h+< z%!B_WG47yXHlGw&ga3PnFY@_c_l{m{^1rU-$srmP%w(jOt5XVh@5A>!uNN)J#N!ZC2;k4lApKh+sPTpOfU;q2& z`1JMpo12T{tE*q%UY_3E{B-{6Y_2Lbg-_3~j(>c0c60vb{QCU(m8yJld3Jn#{`SpI zP8s5RFtlG`$}icG73_~zf$jcGEUg8PmYR7AvHcV>*e*)cPYVP7wV(O$f5U`Nxu?$l z|6kLSk zo(Oi@RxEi5)(BLwot>Su2mR7ISpBgE!pzvkrOnxgkWf)IpYpX4>MZosAg|h=K9kx4 z{2!G1+vaRwt^8le|FwU#cd+6A*74Z+Kc%8Cxe3bid|LMpV@9U@uk8D4a)$yNCeXd` zdawCk;4ul2Q&F_63l65qeSRJeQA6?>7{z3eOY@5sqPMx0**unK0sLQ|@Gaa<hQ*74l+Ru=c zb8b{nx@SvHz?|?@X9-T>L+S42@!+&S?Zv!~Y-d9Teh!_c#3i zdLD7RN0~_d=wr%2n=;x4?JKp`SBSOwyJY3l%$NGkNu!sTBBNH$6k{o;@(HBnUgaG6 zot?Z}ISfN;NFym+U&RIG=d|L`5&&m2unTF{dRV=*Dacny>+*Tn>P!YF2ztKEGNaMa zKvVr>G5O=?l2ptM;J4{rfjx#=){DPl<_j%lZa50-#~*is+&daiyV40JGd0^`gaZbi zIoE0eoF5@YBA;Bln>ft~yxiUqfY1rtJ3ONC)C45yBJ~97{7y;GnYT-O{kcG(ZYOD_ zqrB^*xdBN!wlr1?OU@JRiP*p`XU4Xi37vhVMAOaJ#IO{toBdoCn!9+2T;@Cc-zY!~ z2@ex>E5P>#gE&xGAe|G=sW|!kjkZ&3iOsd((bed)6uX4Pv*Ns*IA$S;qS!1ASfFQC z>1gt}P7*H_KEQzO<0guRC5lEi)JR!i3rjSOvIq^013Umwh_?YOF*iUi?ncSHiO?I% zER<-V{HDDwsDuEskR_@lLsZH6GP4vH?Nc@a0}510hI;+JRydB|(8Bv(}w2M(Pid8+;w2Nblg z`Tt)c=(9;X3E>|{{`YN_#-e^MMcRakY(fX-cZr9-f3|%;MhtX1GLujzLNp`+Mf^?3 zS!B@Z=rDF81YOQo8O zEE(%%ebJ#cW%~LEO%hhIG#<3N|F#8@UNB^`CE&1*hing*Y<{f_mTVNX#$`6X=VDJM zp{LbvBXqQSqeD@vpGhEU-2lbAl!OtR&tv5pQPxfkfH!_CO`iqv|6HzsHTb`GbdZn#c=+OQ zlmBBa&&OgMSv&GxiHG2--18Pj4POZ+UmrqR?EJ4Z{@?yVuMq$1#lhzMujOfa|JN`I z%mwk5;SkyErihu1{~QX)bjZW2^jjowE1P`DW1YH za>Cg6r4<^@Gy0;yh4Ec9lz&^X@cE)`u<#dqr-g$v|5>kI8!u5iP;6 zwCVs66yZX+G|GyG6_#9;{9B!}(Qt{Z#U_agu+p-v-l{UQjdE8vMQD@DoV1I1=}Lq+ znP+;{-k=$Z{>oxs^3?Nx4kwa$9FPbt*hiiGcetOw|L^sVHu3+~@@#<%$QX(O3K#)$ znZf-81z;FsU*v-gp?e3%h&rt;a6Q3PT-u5tOmu>LA8-mU4M1@)-W4vP49}2our}Vq zz-w)R0F4E;0o!4OM)(8rB)<9o?KpsABVi64N{~Q^BH-fyIW6b(>gI}(2(`Ar37Jkw z0Dd{S0v?WN%Nb+V75|sux18Z0QCIw5znF}>{D1lsj=D5%IoLz6E6YdZ_tx8A-<(}u zeHBQh{`_B_9iP5FbEe+HeN>+R{!#z%Fn9j@FZT8}`QO*_Y-wY0@Dlk`AUrBst#812 zKpFIXnX~5}4ME~u!%>3d*1&sJ_qT0jsHllBiN@WXR%>eu{EQg5L?Hp^!HBe4x3{-0 z2`KSVi<1?=Sup@T52%`eH$;t=#+Xgwp@T`6m+#2EY1EBF4>H8tZMAN$5CZR$ZxSlr zm=^Oq;!^QL{O^Vr>h1_*dc~6gkGQe0MiI*H8!-em6*@Jxnxbn0u>2s?8u7Ohm5Kb> zlKmBY3b>sMw1>2eexiJ3KT(5 zRnp?J$7_+VNzP1!reuaF&;$Ng;-JHegd{)#qX1G$Tnw2;>*>^p*%jSTbA}u_u5cVd zv9n+Rv_BJ2G6W|~5CrHROyTb&0yC*~Xtj>j3)z8s&3^@B1h##Ahrq1w>^b|O^8=Xe zIX$PhlgUN}83!do08;>i77qAGAj3l+fnkK;9fv}-7a$}6oWdZ6K7g)}nFRP-;F^E~ z-aObo4m_~$^!J?p4xc|v6S$N=2bBlyhws52FV;(2IOHv~TK^_7n82AFw&#IPXG9`b z{CpsB1jK^HV#3nZfpSt+co;G=CSJ592jQii9m&qBA>4QRCV($wou9;c36z(}gIs9B zCsJ0FXLMJ9aBGVHiHvp&Ldazbh9YneDTok1Ctiz@Y=ui*-T~($K6-G^S1TY4gb|u? zitIlCikR3gC4KFJfT-naVVb(tkU-C3w)J^h7L~B?xA^W53{*fFf*!9~HWabprR97q zCGS@25nLn_QujQ9Q$*behk}i&e|UK*KF|9+wjMjppLxnIQr@W9dIY!HQ~9=j{BPpv z5Jmh{1j4bauKaDkw$ZZr0=FUYPBD#Qu{=M<-WV}wii6`B#6BGQ=r--=f_T77%7Q5l z@HCzRsEZch3b4NjWaV(d5AOn)gRBS_{2<_p11n^o0j^ZFd~5cTgDWj%Z+$tqQhIsC zBFNDAp`3J6uJyV zTK!vEIfW?E6xTtNW7!vgi@|YStw->7hN1|2iiy^;n`i(qI0p9u7~L}mK_|6C*ON;p5Ns2H%b~whRel9f%x0LplRFl0+)fy$XH-+=MfxxoC9VanmWU93$wXLrPF1HTXJj$TCe1EC3$!eZo*x_2m zI%bpOSekp}-K%J(2Fu8Z$WH@?sxH{ZBYj20S9JmKO5rk--xz|l2wX=-9KMtD(@QGd zACafy%5V^e7P#gmlUe$h=C~=__j=iMQDXuvYKT?QZCfs!+e|-7rO6Sf<`nJ8brxMa z9k1=Gj!ry+6KOySfv!`@Mm&OB;dVOzGzqx!X=EyJ>9OZw$(kkr$xF`>i2%hOa$#gx zz%0CYa=r*<$3BJ>)#$Sx`Tn88{grAO7!vO}1zp74E^mis%P+O3a4_bZwK*?H!MUtb z*ZOknsNrfrdo(;5+A|ZzEdWM&`m-dC*Kj>G+N0qLA)D0jsL6b}E(#g)akZT;e=cE7=P3#p?*0&l0%aw}m*~K_B2W^f`TV zq2FG`A?3*J0yrXCo`_eW0we-c5+ONc6o~LQ&Z_Vxt3(2s zCAofem7VdD@+*N${adavOyQCmLSD!LuaE!DS@z@#jgvw{=1}*D6#Vc#=sW#yIkfF| z4YW$0uH|uj}I}&Je&Q@uq3uTZ1>>5HGoZ72)}ar#NJx z2QHLf3NFKHw{Cx3Cqs!0aSp*vk0@%nZIx z-@+?iORKs#oU4MX4kp>pRe|NK0bN+#^sQr8R|;>4h;e@&FDjobrNu zEQRY;9dl*julU{)Ha0N8Ba~y3S0LNF6XeCdOvh2pFPOTvyKgB-s)B84L%}96&|=v_ z(z5}VCCF^qzJEtiF{CKpTa`(TonIN1=kh+BMVnDXRZ(Nv{3%G-}@P_WF5`m`1uCqI+{jQ7r&ec7EiVy z*U_STn|q%X=PMWd_BIPwVIAw{Ybm1wdr046dynA4*jGjv;_^G>_m>EIzebp$w}Fcq zBeEh~RkEuA7yBiR2%V8)b8sz#zf8DBIDkI>!%mLM;7Tjy%iq2S?QAgbaOQTxh%h7= z^)|3ip=IB5@n+^)(f=bj6C#a5_nb0@q5mLF2QPj(aSG#D*?iG>cu^`=KLVN96apF# zfrq2ynx$|ImOPXNxNsoZWGmu-1m{_lim5QGQ)PF#m?xG#SWthWTs8VBe)&L^kGQm*7TIZWwpbohl{~+ru=Q++3!2^n-Z47 zWi_p+hbwDlv7}&NgUf1KQ4d$v%wkz%iVR#8>VzU(TwW?@62SieN|&%OrI=SRIQGR2 z3`2%UAlwH^$}fe%P%vvqRuo}H!3;-4O64;cVX5p7A*IsrSsRWmD-Z`W;duH}L<+oZ z9xf|HqlioaOs7)r0|#6odEa$>aSlAW)!> zB2Jx6^(72(ch-0MFS=VOn9X1$yY0;S9hXE%x|j&E$bg3!#6ESvNsip>snheor$AZ; zH4`Xd_WGY!1lv<(<|Vxr{0AW8wFuE%*iZ1vNrZwOeMq_BpzcT*;q{^Cp+wVQ>;?NU zIEE<>qxwGNmAb<%41#{CQM~^xJBlRQeTAi#WfW<(r!bI96nH4A+gFd^3I$a7!8AJf zFz}?FKzO34>_1>zmzqQ*9#6n~7W!pQS>&D(<#~9+a7b@p;N7qYyLYs^6;m9HJ8XhF zEc830I1rM<&Q4Q&qXy4GS2xnHaBx?3P4YW(nZ=xj^5I!$#973=E!Ans1(9z3<-5zc zlQwPcgn@`S?}AmQN%^I8xdN_VjBo>sz@=oH&jeSs5-zHM$6^9lUDiSHY#D zi|4>vaEnP{*pJ7!?!HIDbs>u?V&N0WsRi^&FqXHo`Y`D7OYM8kL^ry(iBrOr1k4I* zt{WaT9(Ybc7~v_59x5#mmW9ja4O;@2C~E2n5_Ocm$P?eFqu>f+zmY6l0OX>|{lW%F zx&MU&j(wlpQ-HA0oWcOWD1r}M9QY{X6{XG|pTJZI1b5%k4rLEM+SvspTZghpT#IiD z^AuOcDiJ{tWCdqw%x&fstrfNfXYlIu_yWL~O;EtFTWY4VtJVNe+>!X*1lnV4fsb*ABFyUpjxzQEV z(MiGxT`>}Z_h-^Fvr;3-AibkqeW+00ke(oIVPv}ivX#I6PYfTbFNCZhlzQJ5 zL8$h=eFV36_#TfV>~#epm9^)Cr^4$t4^Q@n#|ls1**kLPclENsRpCgRhpWz!c1hZj zf$O5ek2Viiog?j%cRm`fTI10at~7pSW$UXsa z8&u*p@3sh)(9bOlTLW}vgGjDIvi`RUqI5$DPeQqKL*(KSM&535+ZGp74xuB40C2!d z^J>(%%%Ki=%jLp*Or@NWeBq0IKf&%K(-Wp3iUT3Kh)#z{q<43~IeSh4^eF)$q4JJ~ z5e4r9IDYj?{FET#KnTl0cE>lkIJV8+J!J0WHDVKD?E6O`-|$Xj8LQ1h47n^loXXvh zI^b8ny1yP@pS&zisz(4+-}EK?uG|h+ug=ac0A{kB@b%<_N)NwSq5x1Ip-`;KQIBI( z5_odFqSagLm^4AiDd?}{)-!?h1=ENFF}VWC?4~+#$`Qx%1b6nG)Q0o#=U$!6pZ0XX8Q2pStzJAv}kq@p(pJE;kw zY_Q+!^+Ze@=uL6Z6*@hN3Ez!fts3phF5C8rrFS^I6a_MeUm!>+j$;alu&@x!uolrT z&;IM?-Q_ER9{Jy|&#qs-JuSI~OtH5O1RP*?EdI<|ibHctyt8g+7q4Q~IbUZ7i}>=5*!@h&Z)yd6@DYbt?AjEkAcI z!!yp3G6LX?U{5*G`$+FMdFz0@9CUN(Lu=s@@eST#88way!1x;(4OhhM))+erh_H_0 z5WhQLL`ho$_j_j*t$v7ds_USJu!XxX=P=R*y8i4u2 z^AwNtg#_>JF6Kxo#K09ntmn0sESqqvpj9Jj74c`4w3U>ePr%8;m>0XUfK~}#&A^>k z05jmr;CchJpAuxRwZsBI4FpmAxJ!IrBx*My-HrnhI>gqWa;1~aO9eKjfFib|xH%C? zK+9Olbx~~6E}Mqk-1T$ysM0$HEn8ks%&L@0@= z+;Es>gR%<$%SrzRw0}AcK>0k1*we-HdpxAXkD1E*!66XYL1b|eTH*t8`S+|W%05iCvO?}tWtDAjLt3G`o-uJgc3p8oN}8m-B_BwI=g&x{Q7KH{JJ>4y889)<>~H9 zB<;+!TQn)_SEZQJt4(yy%FYI3w>t4Tv1nUHOKcj(twdF9!nJ_D*o3DM8>5RaJb4SN zj~jwIKMRw>wt7Xf3G|A>t< z{`x7K36V|Q%TYF6{EN{w34p6&6xu19aUiYo-nuAT3J0e~Z;I|JRg^86>GjdI6xB4Q zYE$-k49u@yUta(>LLLejhCX!~UreQN#S}&VLe(MkBwTmsL90s>xC8`t;CRCURMXMs zzZw9R!WBYF?@8oU!L|@wx?u^I{#65;w0h?Ut;s^pyN;q^b0Mw%9u3#l0JXu|mQSGs zs>#}llAXF0HsETowu`})U0VrP-3rSsmvN$Xs-0^e!Lj~c?NbYdL8_-M3^6Sf#Z<5l z1&omcr_^^KNp_%$*0~NmR&0gp#|D?wzuWK5RG{_zfXiB6NwO9VSGgNVVZLfy8dj5p zLBkc4`ZE|wD->V?dq~D@S4Ke=PQD4(sccRE|_Kkdo+Y@#f~&#*GT{ZPk?**+ z{2DSB8A(D2BOtKYfoA#lzm-mY<&%TU*x)wsYT+^u$ntQh49E@GFmNiwgCu9N)SswZ zNe>Jk%vcvnQprT-YkJEr8LMdA#P8973mP_^(U@`stU=XWi7dZVMLb?;I=|x6lHY-D9j2F2iZe$(CQnS1b!xg=dxp zu1e>uWrIkN`@kpAJ2j22ZK3(f;@wFkmWJaJhBms@j%mD{Vt-L*-7>feZo4+PsvLHgbk;3{s}Q8s23M8C?ve;NIpaEk zak-x9ZPxHHi3*Wk1m)o3^R<0@iF^bpa?*ROik!kArx18e0=|DM_#qitUlpIq;?f8_ zjL4KTN-+f_L=og4mNECX!4>2$=M1=#0n1x~9Eg}o@zVDt)?JOlNy8;FRnDOG;`JrD z2P1Jk6)^t+T%^;aZZWAF!Nf1%5#08xL%z-fS9zkdM{uF-6V2|+TL-t@DPj?J=j79@ za8Y-H>?)^6a3#K(DjU_Wgp||7EL;=DD#H2A1sC&G6f4XADrTk{`oz6Uv>5v8EP-nU z3OWr}mE!PKn%c!s`s#h3vP8zldz+q5se5e#t9>2$-nqt8L}HfXB-C4VeQ4ZR%l(Ea zpD~@}8Hnt_^KPwDxVUUKL_Q7*!YAicLl*&GQ}1U#W&h(;tr$$oP*gpH$2*>y7}RXki3w6i7Dq*@<{~Dxq1^%he#xAAif7Kr6-V#iaY0@Cf7GR7&ox?0sVrbt1(bQr!*<>% zq)aWC25VaZkuAPVDMZx~W@;f)8l$fIx0CIq0!(|@nG!JT0?*Wf$t5G(twB4JdA$8cm5vhyv(5}glA`4x zUj)oL(q;#sSDs@Z%=CzEqbY(|)f#j_+|s{&Kk1Fp#O zIv1GFpD(aCRf>9RMyhhwW*%-P4`$t+L%y6d zn7l(8KA1aTPB6dieOq}mPyr@`(HOCd46xO+sllCd`4GThh75G|NFeOx!TFGC`|q)l9q4_lMBE6DG3NU@mad ztKx+|W&)wu_Ng^eXw6hDbcCuT!bcD!LSPO^6&V6hB@o^r6gCF4PExdh86b8~qC3g8 zIwCQZO3!_p;!OzNn8k#NtVoE5gCU6n53mT1M%b0y83}x8u&Xa*j8qeJLA+@J^Gre6 z08A3IES!wgK)ImgS`{x++R-HGz*G(}3yZ7sT{T?v1z3-&+~aFbH=YHoD*5=D^Nr_F zct!T{H76X;9)(6K9W!WE_J|}Kwm!(re2`j?j|F}aZs_cTkmvH$@e!P=#FoP93AjE^ zLU@ZAc1aM(RIl@K!j0g!U@6=Df8h8Uv(%p{Jy?(6*u1zFN@e|3MlFzaeZFz{Yr{)O zqEe;dRanbSLS*qtLsb;azQ-)^V~y;sjTn5hxrc<;g{TKqYqqOBa}I4YCmN`BWViK< zYCF#CzVeBRDn^C5R$IX)Gc*CD#%X1pB-zq$B#3gX4SPN9f+&!^k@!(-1N| z#C{f=BqJWGg1|LPa$r)0gt9RYiT7N2ril5Op@@n*6vTn;JNUIMQVUQ#9^Slct)2FGdwZLmB{Bkq+-720 z%*`} z@)c_HS?U0lazv(r67wdYx)Afzox!N<vEiKWZ%$A?Upg{D*}dVH$VFd1mc{^zO{ZWmM{t)nfr(=69!M)lj>)~)XCb{E{b6yJ&zF}F#|vs~S zznTtEGW))e$oO_`-DakUZ`G|Wa4u6R83Sxx3+x8@e_({}!Blv*nu(XZOg1JZnmXV| zf%teuN~Q>?lQyC!|8HyT--Ufo<=zI}am z{qpSHRkA?Bjg?zsbL3CWs9B23Rasw20FvB?B8CCyX+R+2da*bP z!2iB_`$p{`|3RE}!+i$pVtkwa2q`)`04Q*YhdjAgw~;e;z|Sa(FxzPzg8d9IQZPg)$XbcUkuR`@3n@o60FS@}iBlX1hDQf#rd03+^q&~|G;wT}_8Yn>6tL(a z1z#yC3;K;Y8uL2!%d$rSkaidq1hwH+xYY74m58f&^Cn zULFzAu!17HJ|2$xT@Ocl>CtKFT_QIUrj=91JO2CLPrU1NzSO~#Q^mXZ``%By6aJ6x zCYd;KLUr#+-S%+AC;lKr7zzuKfbCxBT+w9R|ToTwnEV@k&#x=mbAEtdcl{( z_su{Lxu74J05NKk8Hy+%vGm?0fnu;*P^l4M>e`QkiomRn@KKh#*{HxKgd!6n&V1ZQ zm@yQzBtew1Durz1x`@(@Kac}XWKa%$_;O7tVFY=i0jboML7-b^6!b_1ibkFadgD2* z)-i~XOQus4cv61GRI@uIFnR(ZGFEw4gjE0$@?uv@-AR6vObksvSFLPIe9hu0l2(}< z&4dvNQRF{>DUKpBjx_#`A*!Uo@?={E4+ca0i{woPVAgNl;lLZDF-KZc#2_b|oOVf~ zC(XJNwmUsYKxqvXnJa|F9e{Sf-O`QSeM@uf#sEBaS^&Iv9Ot)$mSs;$5(TOHs93`I zC1*tjv{?M4_?R+zPLh0*Br>+uF5k29c%mq%+91fYou5|e`2n@(Cm#~fet=r?0BKN- z(*qiu9?lymA36uTqbf20MS$-d+1GAM2gTC1B4z|A6?s9QDW}y^!Dcu)=fVqLRvA)- zH*SWx!a;H+CWHrur^qTqD1_`p)=P}>*)w)uX%7ZE}mT(%VKO55kJ_(R8lV{>7L+;hRB5+&zKXYsjb?o zAdEz~dvV}^kMGh*6O!`kSPsItCxwGgu}E7ALylY`3|YEoliL+Wz!V0;z>RM(1J$Iy zKhX2I=t3wPM|@MJfzt#Ayvw^B4XSaH1k!?#QcRShhb%5ChPE7l=bTY|4z>~zU(){( zi3Xtbl(l4L*HhJhIPQ2uhl8we7XT*cQqnUT$x9utA3!(;-q0wM$P(-!M1y4ZlGj3Z zm_Bs$hyVB;Z#XzS*xyTwP!tV-H|#_RdL-~4L^zS;v#=II|C^FP4?{nCM-0I8t9K!a zuFm+3U%+S2_}k}6$=@w?`3Q!PA`YCKXH%Z0)SnWM&$>#>ASPb^*;UsMT@eZXq2ogR z*C_UVqme26@HRxj2?+u^06nmkAc)zabMXJqmOS@-q)P5plypfDyoMiM;Su5pW&rvJ z2feDM^}MHx+!N)$>+OI4owL6SzUv))|DAKd|2X{qJLi!9arFIn&e5(Y`hCA=w8s0S zF7%N%OqFr+*#iF@65{`$RO5UJ#i>FbW|?zuB!)=U9J~~tT-~Z5r)5XfJeeL085iAn zqwm!uNYlPSSJ?7tGX{r5V~Oc7W%AZAi}JRVddN7J9Tkp?Q;waC9>_c*=BX2Bt7SV^ zmSeRA{wdGruD;Upe#jUsq;(u{sYZmyPnK}u&YUNIwWEi^&H-RCf*cEBBRYCFMIA#+|k7?nA)aGlwv%_lC)U%MU zc~P7lE5w3(G{av_5cE)_DG2|3l720;HiglhVmie>F^|k^_(4cEq1cvVRTa}yQ*kTW zn!@l0X=6pDPDux>aY-;kBF7FRQvE8F08EQ6{FNXdhQc>TYV^rU&_TQJ^xNro=2Njg z`t~XN`d2>w;}aCIOaWsp1XN`BJ3oXfvTk+^C~%{PP_VPIav4QsKGj{d9mi>BK8U;a z%8&m-57}?CEMu`8jj&RPoWPZY=c^L8Eb(RkWWrWHQtcNNwajpoLIMhqUnz45U^P_Q zsb3pqNi}5ac7pt=b~e40N;$|fQ6Xu5DS1C4NGVI(UxiE~lH!#NGbiJ;l#ByX8Ak*w zT#KMn2)SLwJQc1;h@0NcN*AAMW=hOT$DEapo)wdmN>2GbjTtCwP1iURK@LQ@=gTu@2g zD~DtEVF3D)+$x%O!&k$fp_?h96poR}DsKtTl-sK~RJq%(l!=%K{*fe=N({%SDku>g zg6>2$W7&3sy;C5|>VxEJ$;BbZ(F5@OzyJHcwP+-Q6%N4j59Gs8zl%dI?F;v%48_S@ z0*M6>W*IH1XS|trAuS7Mu2p3VdNDQ4*A@F#qM(zArnYTW)FGdA`OE68x6?ob8gUOF zYI6yoKT z$LE1zf z)0iPHc?*01MM#5t&Zs0P8=Mk1o}xe+6Er~J0s6qw&Z-}DYxIrOU(R{5G73-UyN&xYoczKGPysET`!G5*gXN1p^%s2XM9&^(E}upi)tZr zyw$eRs3hGbyw#RE>_M9PEgLFAS5A^U7?n*vNM}&Z)UHwHR;{1xu~V{)2?jSU8%y-P zmeo~_H(g{|T*OLDmA8d;7R0xIbhLNy;@e*5=c{JwK`xDAA5BcR+;3u@58vK|mobM>M{< z)&1Y+YKbHPH)BUET&na`c3KH#5k?dF{Po?fvGp_p`M#%%t8=AaI1@{W`jgQKBe|=_fCI=w}Q3iiQ(H?!ddtS1Q=v z2&-UGxw6v8yKQM9!77NlFhsy3v2baL`Kcd9(pN&AF&6q@6bC}Aa&m1k^npu5A3uWI zj~^2p0@`ZGj*{KlzyY5=-GWb_p646Dj4y3@3(^SFs4t1_s%0 z<&j{-nT>D2dElq9?}dq$ve+?rMCB7>AXP132&|^@g2S5=9JJ^_wiZ+QLxf<(GnH#o zsC@W`L_;SHHX}j|tIaF~SxQ_6f-obKP{2S4wnzkR5Wtu-zy~kH4jErTVQndd;G7(S zQXbj+5upBNCvD zZ-As)WQHOxf=OB^N{eL=q|qW70jxIEuJuR>{ThsT2g%a^oW?LV=#W|$K@B5mT!{I` z^f-@54cSKT4LJ{bK=N37dcdh;EF%mm&7#Wmn^5$@tu}HLUX;jlz!|4wV(P{4QQ(0% zlo$D#$gEaoivK`~!bTgZW2(X9vhKZBSZV|XZXJ;@ z-uW4^TOpN9VBq;80{~}iX*?QXjoqe%GO==0TFt+d6kOQ-C7>x?u{gM4ghu!S*w*`d zr`0+S0Es*qblF}})j`hK*_CF7gKpPwg3*oru(KIM+OHNfbO$?ZJn5+PK3tj!vM57 zaP3y&_LCrr|GDjavy=B4y8D(U-a;OlwW=JSQm-a^o+qs+-;(<0iClW2dq_$qjC!&W z+&Du114ts-Br0{o;0^~^=`<9qoCT!GHh7<+E)PX?Va;#`KXkWJJn5lmN2xnS3}3^} zR&eaX%|`0LwX;$V*Z6P@wv8b0dajgxxN2AWNbLyaxRJRk_66VT{VV-aZ!mxL?B<4#_hT0^x2wAB@|ri9cDhNOPiX@R`nemCB|d2* zicI<;;n2&6_YT^KN6gV9LPDl3Gl6>OZ5dJ4dIfJBR)C2HO zvG_maN1uKWqd5;;KlYGD+s$@}IWADBVJfUk2hkDxM(Ni1i#IicmcqDT1YZkFqTa!Q zG%_%q)T-B0Vu!@{Ozf`2#jr4rgkwvSpItE$!7&2o0YlLUx(IBaT%GSEN7{DyAvn1@ z2VTV4eIR@xBE%w$X6YW%UA;g4BaRUG7kU7{2o>>#YS+LN!c#;b3Q<^(wj-|4J1)f? z*;RrNI*^y!X}o0F1_cv?3iBMrf$rHhsX~;ZSPLu?qFmFCOR>YzIwr}bN}&;;n2Te^ zB0P*4GK3)6nx3K)MZBgD$;h>|6A%ww#6us`i5&DL8Vz!y9K4%#+z4s+KfyZlmDT|Y zz=a==JLiE~;T)@pbZ)=*5t5Ma@K0DDnWwF3*iB6Re=s`tc#P53`FD&KB7v$wn_!Ij+_Y}su~DWZ-k;Bt>t+wx9z_E(BT zhC_~W4Wd}mnjFnNd}s-&L1ze^LTS^hS?L1wi~uqew!qN$MUnFhs!F%C*)sSyiFH(G z<$MEfxd1N0-F^CWdn*YVGqUgr5QV}m4M*jal6mQVsh01?RDBg$RRAXNCPXNr1CYYd z(L)=2*Zb~37xRUm><#oh!T5q_A1~gX-ke{2%9MiMjCV*;X+h2*+kr|gWB&F8AL)J+ zby78WSKuXT@oKg^c-HsP>-Bm^2M6NcUayz`x7X|K{jGm+^kT2y@AZ2}f9vfX^pB4I z272q5fS58E{jIljU-`j(CC|rCUmAkV^Ch41^FM)+hXUliAs#yWAnjecC;XcrD0B2t zcm7`-9pujc-a)UoIsfZ;a%vqHGU$`B<3h%KLWlh~C$|KWSk zclM5)UZ)5C3Hr`{pa1#XT9wcN@Np1-FutlS$v?ZCqH>Bq0P3y1oN=Ek+cx%dhdxk- zruK@FOd8UW47@%JVN54sL_XxDZ5W2h&z{qFdMyu8H^Smll8fP73JU==18ge3>LR9- zi->$sCi+4V3r-=LlHdx7Qo2MUY*VMjXZ9CkV)eW}Z%Ke{XH<#y!e3loX0>K1(0fiV z!-vdy{tb?@`5vo2|NX=M!G8YyAMNjN&i^`|EpTeOu1Q69{v0?hGmgl=k6*v)$O{Fg z0yOXfr^xkTgvxT0;XtdN)2r21EA3uNyGH)6AyEQDdSav%o`^4Z4=G5(fTuEHs^~ry z(k%zBwFNAavk{WSCDUmfVD`|}XHW{x2tjX%UK|}A9rTaB`>wzD-FJIE*BwC!;eHQ( zd*~j$7@=?b-L$W*SN+bb{vjXOiK+5xwZ=XfLSNxn<}0U>kcL`|kP{e$c0yn-HA9yw zdxE-xrRWvMn^s|?wFQo)Sv47fD>PMB)09OJ2V9VhgxT7*VZLaYq1QrhFvWpKyHqqH zZ5Rz4Aa1ZC?D4hSYHf-5L*jNtYOo1yuX zu=$)OG1R8gc(b(yVDONbisZ*jwQ5aVb={!{ssqeUVJP8#8}PoRrcN}vn@Q=Lcy-Fb znG+ym0IO;6P>`6{DeFcqrtI?E@YtVfBh*MZhLAX5C_*0C_F^f3&rsw;nYC99 zFA}zxJG*KevcB$H#|pvfGD-IMh6vTy7U=NtMsXmouB8mF59Ka+$$!eDB@AUK6+x_c zKgM!$R50s1dp)OD(xP$%0G*B(J#?Zt*iA3>-2acguWgRoMiTz~@K@lWoVCV}$QjAD z6i>yu(#gt3w@#vxoZU;gQXUM6(QrkP48f5zYw7B~-@m+H@)WuO5Fo*MktTAotND;v zGk9q<8jXHKcV&$wEeB&pqG`@qW?g)maE9dl&oEcLwL{_y31e^9_qJ5(99SaQ8nJrP zX+*Q;trB=PG%Tj#0%A<58*bAFi!Q6QPj+ihl?*)`40r&8*Kw(6n5 zu?9Yiv95W>z$Ze%M>z)KtYd7qlaPgkRZ-mYbn7AU^}qF4;N@&olbsUqiV170=nh_JKbEO{SW?m} z9)7^eV0vu^N6|~cRBuS+fH8>`w7eIvTN2yO4MXf*lyuBhF)H-Caj(YmhgaEVNDtsw)`fg1U&sUdpahAssS$reszwd zLdcqWGtS=X@BuB~HAF;=Ux$Ym!0RO)Tql%#`|jQ8xu@?bjJkW1%UfBlP$gi7 zM(|n?Uxpii{hLczTs)dyqPQLyo)|F$$xECG$Akj;047#3vs^Ic9maa4@xGpN?bBhC|6I;87*1)PJs4JlCs)W>YzSH z`%{_4_nMZbpSV#LN^Yy%)jlA?UydCa!jWY4JAluLpL3bGW+8^|v>Zjr@-fAdEkL`G zkC4?-zES(3zxZ6j!*UDd;-kvHt;YaV-huP;x4IA&s)cywy2WnEy>yUko*^H{%Oqoh z1=4bkMNC~;5)6A37o(((rsx6( zI+VkpC{o8X3J3&LDl%J73Fk%4dCBJ1-w{1(HrZ0PfFBBcKhUvS^Eo$Vd_s(RCjbZ`s?dm<5^R?;Zd9k*)$W8^;x;y{q8CGQ zJqUNh=YeduSN|U6S64KPhvdEA=4RzQW>mf@;Mb=1(ZZ-K8tmj9hoF0`zQb_N_Gu{pXPD!E)Sq~ zpR@@}L2W#P#}BH-X&&Q3zEv-mrN#px@abI0?+w5jmKe5i3S-G}xYVo}1-V5T-i@`@13%@) zV*pteOXXDLc+bPg;+BWWKsXZjA5X@z{K2faC^2=TVHg%NT`9iAGSFh6GP#&mj_E>p zk*OeZ8pP%KfaXIo@I27pmZHpQfyZ(nMOAA)= zuqu|hf|vMzU+nL_Y{q{b9K3k+|31WLi=4^83h&dWKq&w|dLC8Yoe!#VyHA4KTWDw) zYSFW)f;yZ&OWmy~ldPH${|nyt?=%3*;?}>tE%LV{i}R11$P_+eG5u$r`c_*Q1(2tq*_Wycduus#!hZSYUQgvFhn=F}>K5KUP;OO-n!=gb_@d)60Vnh1tp z1MzEMjE?vXq;hSB$p}?XR))a7;tMvJu&5dm|NH!00wBItR*50;kAIob)VHzK0fw?{ zTYHS&MyQ8XGU9)=8}7ac20jUoXhGFO+3+v5w=Vp1QYp8d7-!5X+IYWJ$NT_TDQD1f zVI%jh{~5iM_LAVW|6d&b!+K#)wA-2tUz0eAD*Y4qRqF=gcSWWypJX?#^bn!pd4JN4 zHxp?9;|DSK6)P{mMTpNt4Z!R1W#Go56Vf@LT$?#?3(qEUbwr3ZH6d0BX)uMBe^WNY zhRa%6grPf~VX#)HlPa05T@=!U{1ki_LRQH$DD)?3LPmi>w74}3xxF>rtKw&+?`s$B z4f$o3SI|`H7DMB4oRC{#5M-w_q*Gw{Zl({^pno3FL?L`5BYU8{37 z-pXW;Kyk|g0|w~tSA;W=eFJH5577GikkF^bsfF}AMFGdMPV~^R3nA{OyP?GpLejoy zpxwgLyTWFpKJ*~iLXg%$mUV3Ig)2J{qvQIp<3Bo(_XQ7tU059XA<)-=&cve|huv$s z4=z^%PDg5kDs%W6X&4O8f?<5(%iljF@oGRwh_67zSQ!~@%)TbH zD8%kKerr$Ly_i$>#Tg(3mp3WOL%H;HXUv zs>WN+gKTuD^>1Pe-0a^y-#_&pEMefrKdbyFdX+jvX?1x$p#SI2{=Y_C3J=OLjoc4I zm%RUedGNC7|2uej@RkJ_7b#@E_l2FmXrD|HPDKErZ9_BzfSIr zvgui#GDl*wl}&y+r?=?Rt*oco2y;WSBon8byb%3M%-X%sxw6zCN*H@&UGuQ%!(}g@e6W<+e;tzjmoJ~c^az=7 zykj;roJ!Jpmgn%41w+2-eaC#AI&bq+eBrdQ{gitVkf&hNe3O(<)fYg5w3Ziw}|T?Tmb+`0cxQr;f-7 zC)|w`r^{4M<4cG9zcKx9K~A7$`F{qxgNFXMzyEUYQU800&*#zqXi;z>TX*y_5`r27 z;3a9i#y2fq@<;Wu!8{5ZyhI&d|IG_9gy^%y+pI}j|wo8Gp(ZO+q*7xABR-;dgbswMw_K`=^}4H5-_*Sv zlQz&sNb608u^wjtZfh+au$#j!){m{0H>5eX1+4X2qyttfW_eX`%`obP(3%aG0P7Mg z#V5`<%FtTt{U#}S^|z#&ev4gXLtBCQetqq=MOjizgcmlavot-|x#{aJjUNrUNpq-m zhHIfkzu#?*?0Sr-A!;nW^A$h{9KW@1Fh*FvCnC4E!_HUl8dp>X?&ef7#UR zEe&F!8I5ZW#O&s@ys(Nc2EpW)ID7qM``aJR-+eea?ODG;ph_?+)7MY7PtSh%&!0a0 zc=onujqjj#Y4)<&a6PE}IiPV|igAB9d)He8vgiX6X&|nI*KN^L&2J_lsd8O;!F=XB zW=gnqJo=>-JYDl$12(YgShbnzqzV3vv!E`HvS6(X?H~?(E^P01a+`3e76HFZir^aB z+!?%OW*T#1Fu4Rmqj-IXOc!?sv7t|BXj1&?HCDjjDaiOp{l-7T0+OlP{Vv&jNw?sV zxw}Smg-L^m2BMiYLdUurU$N}kJnF~=yiqI>m@6FXn(lzEKInPHeDTlxVbHCQ}52PfZ3C8Gqaul)$2C zPW*Ey;g03z%GzEd<^+3=%zGjqbtBpNtx+NEJ+(+?vntbXeI;_U=zO$CAlG7 zaG5};4O2#A5@ckzXUwDFU1yrcBA3r%u63>Tv~bilmoZtzctwJLSeu5E-jYB4N$$fp zG?DK;0Ss-^^P$PFNoCdWu7_;kt9!&tHeYGLHfk0?Nu6-Ei0mjYAk%|xjk#c+G-nb6 z8(+RVJp1$UJCkOk!qKVKgWTM-R~SvN@|e7M@nT{7PS6$lB;*aW1K|2QH{8uG&GyTn z?rz?VZ`#6mYj+XnhfA^E?&#>X6!WX}aEe&=xpRsgP2+i;r3(>|m#CbW`4Cmm6l6$U zZa4o%4CIi9EC#eXa5Yig6E>OG%LQ7KWvvd~(cvbsr!$Q?foxDk8_2b^h{mSeZA^!X z(Z=Wkloix9^OYEc$+LOA;@)*jEgca7Nf{la~b_{o) zu)>x={4XLKZ%F=ISP^iU{C9BBtpB(F;!*y4n9paC|5W&2E23|6EPDfydP{gV=?K)r zG*6nPuN^hlJ?#d9=DG*BW9Epr#Kg70*X#E5Za4%^62Q^`I7hpO4u3-smIlAIn=Ju! zjmR~p6#7RET(iCL_whO-T9!nunT^(uSu1p=ulq%;IcaMB_=0A?6`~84DIb3*rmY3- zyi|Ifq^`oCD@@(q9b#wX+O#6q;rW;^KF8 zA~MPB%mzVQd^J&X-WFW70n%xL>_FVcpN%Nf(Gy*xboiS1K567D6U1}EiJxTGG);_r zpc1z|vTl!1O(;{nI4E5=S?~;^kxxnAG21Hd=@C3(*0r@e=Xm`E6t`*D9g{P>7Gk+i zwyXR+~x$q^kGS(5x_Gh)Q=;+a;bAfXEIt|CXQ5UQUhykb_R z$Q9!AR~9o&0NQefs*0_wKJg8|nlG*o-|zLv0A$21&>o*a z9Zjpg2L1p^Pw_{;OY-a6GM|qod}9!)WLwoV4o^?y?ZSusEDnGBF?zl{T zy+!^{11_tHxI~TdRE{>+Uou7VhaK{S{hDx0*(}GjAlhcX=q9i<3zGV_(B(3Z&fZ{& zL@3^vU4-Nma-tJ?`p9Wrl=cTzlomRT*(5J@JG;tS2wS&wZJ#MM-Q(v0KKJMU1uxVc za5w%x*dMg{|I0`J&x3q^3;w^3Uw_L~z&iPP_$}qcxBdS8md78{&%%fknxw>kH)T@t z)g{2-RbH6HjP|j4ng87juX&WmtpC{$_X?ji&;Qx+;hXP{!>f3m zIToM)z5U(idyVtId$9L-|3AoQt5w|F^A1%_P<6jij@A6t*Y8y^uS!@=m7HwL^m-CEJt3)_n@1f_NpeV}8 z`{u$H0xOBWDjDlvOoKu&yuRXm{pL_TFxQogR;&#htTv*Veho5ZNeKaZPdUkCNpP`9 z^h-ls(rVhmLr4zEdxu)*Tn1{-{X#bnR1wf#!+%1K&Pc)uO?j^Do^^3@ZGkisg9sr> z86V+IBBRk%>`Gm4oa7&n(-{NE5~eXoS)(Kw-ZYIUJ)Jh~2$|a1ai{-4+y=voS+|w1#EL ztP1~F>}1B(<6RD*uvlD`EUMswB_s8wX|w`r-G%sJZPHLdEH!~#CzMbXokC>F61l{J zm0-b;m2|J@C04X#9IV|Vl`ML+WVzzR^oW!tLeJaUB0ts@@pBXZ*Sq1~ z%m3$j=M^n0@pF>3e%>MPPxF{4m4=+k=LI!Tc4nD4k9a@IvxpTH@2hyPev!w4`Xi8v zR}Yn@F=H7i*;RhcVk8lP5(AFI)05Ei4mqJhjhdy^4te%WDoL|NOMCVVijV@@Cne)k zZQe>6{2no(Pvj1naaN**jQv`$s6whKSUJhdE95}|)KroXGxNZ1W24af-~av}?`!hx+4p(HhR>dPUy~C{d6czhg?v2M z!(e(DJY*j!CzSI%N`Nx)uE^#>5v^l=96OU_WJ0e6I>j9;7(&VjE)a(wE?RHsHKCr6 zc|OzFo9DAqE7dM4Ra!BCBla5Xz6_)W%nNr}t6@y7MBB6}S{E}smn1`8}IuyKo7EKYklUWH>bYTpmDrv_-UYH>gSz=NNqMy=q zQlEY$f1!eyfY77;hYBRT|HSl$}<;bK$`((fr5Vax>Mh_#WY zb0-8~SAv{lR?#H2wnlqx>pj<-qutKBaD4p~IAb4)dzB~x7ezX^8=O}(i)k5iye5w1 z%s{hvtJUHYh*ca*BEk--#^PRtl4PtBEK}ZSO$k@7LjZ#EtJ+MgHVyz71=zR5U&>qI z-H=l?0l6V>7>~*X9-NaKa{Md3DpE#nyc-gT50U>5?7yJFNJ|_eazo(S*0o-h%sOU} z-~gpt7Lyw(K7k}B#xR`RY7{J!oz-F>%Ip|>$}Q~Og->OHw%}>p-QA&ZcXxMpcbCGU zaQA~#xH}Z?4u!kBL*Z@(oZ-IP-S2iMlj(G4-gGAKPdM4h$i!@(TL$%~zsGVLlk=?s6#3I693L<58>mWC`B>rOh{_b3yvKksrlC1vLK2uYS z=P-5k8pb92TYk-}nI_7e1xik>gT;39B8!YIy@Jkq22+C{by1?HF5{Bf+iBjzm z+JwnMNl%k+q)(6P-VD0Er|i-IaleHT*H8H-Jh^i`Q0u^A{+(32&x zX3eBXDg-D;YH~9$f9U8^?z2`*iK4~mXLr&kq(+hlaLI#{FvCLus8BCBvznYnPyaCn zoN^!4k2GcArCJ8KsKYZ{9HRS+oifX@(Z%%qc62&I=k-D~UgrHhvHQe%xRONsj3)}V z^z;`3m-Z^b){wojAk9mh=-_YMQYMeyFx*V0aiTe39{yLeG+CRTd#STX9i>=Ruz9yW z9`22P{9!lD{;~81?+}D+R6~9}03%xSH*Pd@4TWT|)=v+lvDP~D{(E&en%Y%HX4!VQ zwsY+1ooaPPkXI^b+!v8O?G#DPzVECm(WTrTd^PBlamfyR2#HdI2;@%F>u;uqQCV$> z!hjq+wrTF%^W)IOs+N(MhBxY;s4+_q^GHLO-zM5HMrqX z0tCPI{~(vTj5%H$>qS^YA4*<}O;nU-Psx0(1z{y&y|G~%BnEUt(6H0S4PdwBIKR;% zGS0d>h~(9ImP8G2+AN(D^&XY@If$|`SU|y%3{Z<4j2+kK4<;)cYD;?}?To;E6CY;D z$?ILJ>y%-)m^(saO}9B!{-_p^RxPX>&WW@PCKX<+I;3;{Mm17n!|Yn`k(89`po@?N zQ^3ZAMpgXwosq#3^&mUVEmHs)pbaO$PyUGFl9`#(|A!vL5Muky=6{Oetrj%@e(2n zY*kWLp}I~*8HtT1Oug4mt5h=Io|}b~N2;0KGMNIM)pu)^aw#QtX+m%@Sy5m1zc`@gqGrJ61oi#-qn;UC}wB64-NEWw72ja z`8x~N@HYSE06-XsP%^!uUj8BTqCgUyLpiZ~`{spnf?w3MT5PDAN<$Rq20PkoOt((a z3k(CUpk*6r(X$X&;z}*PrEU_!O^_$PNBJfE11Js!rS42aMkqK!iq9N*iNHY&Ro$d=%O>T9E z>MR7y*=_<2LwN?Nbo`Lm(0QM9GVDdX#NtKzgnqG1K4pA6}AKSGP*c6CRDnp9gx*jQ`(ockP)E%9&y zmEuf^i0H;t?z@bYVetYG;IkeYvyzNjE}L3Tbhc?Rh19Fe=;GslM8BI6(n4UT@Ooyw zQvX~`4`G7FqW0X7gP843y)5Q8z}G2J`m zSC>3l+xKF*dVltl3t5;-mE7CP;$6F-6^m^9gH+_svzsnI=;@RyiWu*&!+HY3Ef4+E ze>5tMqV5y@lGCLTREb?gTFjuki)qF`o#|Lg+dSFnAu%^d>vbx@~Of;R?X%4w($GSF( zDLE5arw4`in1@<8>#%x-5zh#dpoo<V0a%l*q~lrC>7C3)=AuRlh`V!!*pqvK_-h6jL1i#o6~WV}KAaKV z_dbbTLalRWeugqNJn#2u!#wSa7zKq1D*rQ<7&9Owdh_?dO60wFmnh;kRe5}ZUqR=U zFw)8b2vg{ZMd@enm(~sBu|5W_`S$qUyt8})Nd*&a#VtGUVWO-h7T!fA;wZfX;4a*y z{iLb(?ZNgqS7?9zUZ^ul-g0&lB7gwD!(2nmyN(VJboZEgO%yg1IM9*->|6CC_dD^Z zgUVicNWLKxY~-JSH1@ai$)dXlw)5#Kk>bJ1S*x51LGL+o-|X}5T1Ng*^V&nvV-sFg20XvVo*~`=3XEFGNk#3o3}Ei?L%$snd+q+QY`KFTv9(tD=J7NgtZcoHXU3=-HWE-#R^a5a@c^ zbMV&7lL5bNxhe<&*+j02=u1JajOmNrIl(*-bBcj5?LMNPaPCJLt3^6Of=XYJ6c~TK zkgePw@qp8G?#91D91!^(NgK=ov+qij(kTB{c!wq$KKbj+`ocQPElmeIk1Ctg8^|`P z;dGh!B08|T@z~9vF%F+b%|>2DGC{gVc+tI?EHrwq(M%3_dCaT1R|n2MwKs3jvksi0 zuSbK0n<4snq>@;@j#otQ=a%UqN7VsqeuEt(^=nS8TH+XV*H=1-=7zO*hn&>O|5%Rx z7WRdqx*Qfc-C}8!I<25vO(n&nTr=f->_S9pHDl>w{gI%m_W&^_Y;yoP=gN=LCxhZfnLnS!1L$6FFB}FTs#?;Tic~mSnN5`;&1kvfw52v}9CE`$gY?F| zc`OfU6Ie#sGn-5=lY*8sHD2MC{Zcg|pvZSA4={H`DuEf!RC#`sm}8bUa7(H;1Ts<9 zb~YD-6ih<%tWY)uoZ5O&H_2*q22xe~^$mP460`c%Bu(U43YukNjsk6{-vb@~sf_~r zSH^2e)RcDA_5o{TPOg?b$BSnA))3(iAMg5zPfm;`F1OY`QTslLS91Tw7}#_=_conzib8rOjhTb{&YEs zwF^Pff=E`yX&J>WVoCouJ|qAkL!oo|WA4ij{hVwl2Z&~){`p1@o1OGSqJe&03GT?X zRFdFi_8g^VaV5TOhO}LcKx| z7Wpw*xDcue~) z_`CV-q*l~yn>Io5SX=QCKhDgHAA%qhELfYBhd@m|wL94u?JqXb^K5RS2)Qvq7V@(x zX7ZeBepVe(qhObhL%o))ewH3SP$8jDC=Wg_~y$ku`s z;4|PK4pHm1s>CS+Uhc z|C_17mT>*2sWIUw&LajF`7x8n_Y!bUFiTuY&!9avSgCyoXSfga|JA!P34FUd>fbH` zK9dUS(ayv{yYm)<%^HZzs6dP>vL8O^oH$!oFE+DYsbrGkJlmWeUdNHy;Ro1in->cWKlza1+LkW{ zjU^u1)T-_R!QT7LT9E4m&EG6%Ij;JhL%zLvNw`Y;wnB@mjqD63UX4r4B5~ z)%+!HYQ2^&s7sBq<~t)u+E*bh35tRV-x5{;bUa0}1@LAOf?R>EcZ{%9FQ8C&(t)5$ z_$#je<>RWw7PvT#|51E@22b>(U+f1VVsE6sl?iRtsCC_3u?8Fho_g&P>dy?kZ0L{B z91KfMYP;vxJSj!{c}Ln!VS5I(;a6V1abd%BY&nl02W zQqupnx+;VP#S z*ZkkYG2hejoG=I_UPU9pNN??2%W29C2GwN$XKc)I#pA!&m_cZ~-;@XMgV4RL^UN9s zeJkI3-ao|9>YzfCbXJTA|2J?9=83761cxWWL7%MjyCz$nJ>MH|nVlhBkR4cT`hyDF z?vMLBb+JEkQ!lQ1SfLmfFojG8e4YNJxM{X~0m^E!^L=mcNo$) z@!753IJhWv6|hyz8Vii3VmMk$a?~8+{=_m)8eaWL>?zdAd*I^N7xQhq zj`&9RX%OP4^6*=1eL5*QdOwuoI+XG;QxRE86j+ z-;J%NIX&ZzXa6E%cE=A(FRVxTw66C|2SiXdLkNHZWBU}HFaH2B7qJRMlhMmq)dS?k z{-w*|nKQd4>BWKsv!zfsi#2huqOli<3g4RuIBPE#elCLik0R#0FLD;soEPTg0Cpp@ zk#ibLn@0#957x$*6~T1|BBB;69x$=wzL7toj2`Ds|B#f zE)0birBsb>f^qr0R?%PW*;RgB7~h)v=_i7jz%UFB|;$s&9K2Rkw9P;WN^rVdq z6RXB(!^N9rrpO0u_eq(?UVGc8G^k&*u_GMNrXFs(hBygp;AjxZM*A0!$qqV=wN@Wc zLA3qV@?VBRP~h*LP{7;G!P-VHM^DH){xI_7pW*zwlfV%dnvsva3j-(A(L!YD;-EG( z!&V6;j^51BAzCGUp?g9$%1vIP57DrNhP(gX!H{SOG3uU+@@QHSZNEhYIN>U z{$%=trO)cARej!2?I{jEH;GqZwiQ>W?cqS=ei(e0jsFgAsN$z~tYry+4}L8vu4sM~ zu%y|;tg+-5cN^($0~?^y7D2BkbzxD}d2nOdNDQIvSt9aQ5od;q$^xg&3!8;st4oc- zW?u2jvl6XoWirC}r|th3CqC0nrDRASmI`60(jZdAAc zIXid~BsS8ye!RSbzR!CCFLt zJ-xZx^KTo;6fu-G;FJDn!WdJb!eI9MtzSnpL*W_GVVX_Gt>uGbFP>t(*}Kl=7qk!W zbS9f?n{%<^JC0eYs&MM0Z;j$7 zTdbuau1ZdUQCFZ(V~W#dV1( zxlqMV#bnr}z1oBR-Czlc)G413WM!v)&(XuC#R$sX^WM>K0c%9BGlpA8Kl;%_N~*3T z!>A-eZKz=;NA1EIZw4O|9dgA1bE`I{*|5>Ied`^cB~gs10cZBCZ%N!;Y1Q123ENsB zt(o_`RK(Ar+-de>hz1c%2a=U}GQ6Lztt^DJlK4fQ1nVUM)%0*hMiZp@4t+)C!fP2u zE}k*>;!rfmfBjbtzFSo;I;a}$4^+yOD8VCnn0AJ<&-}Gku{9f zXLx?^@9=({2S5p;KmX0cU{Ag5Z1sN)@&BVejPmDsHB6iQCm*-mgJ++C#f9PKu|uF6A1TN# zJ<&5Fg=byh`-@%`=Lebe3XH!Vx$6{7*ry8kioMgjf7f(m8l{SzWznPAGF;A|^B<^o zidnisJ^5=+VW+_s zVdOr+Isc(ybpN4Ykmz>NUY$JacT#{*oxTaLc|?beluQ*d(e`EZTnOs>dp zLtO8+J#B)1y({G4iKwl@zJ6ft>@{zAFW?jc!^9tqV|7jweu7yyaU#V42aN@8eu1P` zcrINkbh1XwahcYt_pacM1J>6^tXCn#wrZ+#E9o^}0zF^n-{k5Tj|y8#x9MuLT4 z>z^NmKl|qCrO!4*(%{6Rx~E?gg|5ZR3tBr; z`>pwA|I)?p33<>j_TfP6E@jWYxA%a0A?tf!dctQO^y@p;djMhYXIe#|``!JG-guPL z-;cix2|}fLVW{8OWukzN)ceAz(f@%gGlvI)7Cb%Kd$;Ka$)Tfj^nF#&%rFyh5N zNljOydT89Et;$**X9j>#*?`5~QH&iMkz2hv@oSXmo8S0+O5}ZZ8HtO)4DG}JNg4J806U2LqJkq~D1a{NB%oti$mK_;cl z8TS5onmr^l)_p@(@pfYbu>=1bNd5Q5=yCX=kmybs<6KsfZaExL z6Ms&=7b>%%I25G2EZe++qD`CPt)zg_hQncxl;Ud#ORUC?5xn{2N5N0%N*L?9Jt0yl zD~i3)QhqIunGWvu7U8TYk0D2B3ToUBqFfvex0_R#+?- z#Ya?avViDH(cQvp3Z;d2@-%6yOygF+Q^>()>6mckP5yH`fz)Id!Ak8fP28ejGD)hLoi8V0kpU)rcTPAvSwS&jDQOJrdL#PUiqe(dfxDvN%S)n?fD{UV3qVQU# z#XkT32lWh@rg~T1e)pRM5%gJtFs-40de%s+MX|ZoD@RO{ID{+$_ZOuTo#8tSh2yYI z)8so`W2>N;!eY2wW1|*@v(8J2RciL8v$^pd3KdM^Gau23!eV{0>o0_>C^bH!V5&=m zhE!IQ5uH7h=nIz0sXT+33<(zP*XKf`wIfTBjz6WA^pg|G!auGOUpCm zn>!tLi9iZ3s1(7R2<&76|U-L+*7OdXtEHS~%Z=zLb$vj6CjkW6ZrJ z`i?~p@Iddc*D}u@-rOac_eeM18xrDwg3|3=(yJ1$7ls2are)L1=Y78Oq2XvI{c)Z>jHLaHe3Q2~Whdi8t{pBUV#n-)q>cNl%N% zhLFs>?KUG~R5+x2%gjSr&|}G$C3B~9Py!8LUBA{Zk{8$UE}Ltf^_}89_|6AzAJuka z{VA?A3Ssmy{%krr8_v;}T1?(%aRb=Er}k?!W0XiVUFf4=_=@pjd{Jcqzhvv>y^A;~ z_1lp^VaDrh*?z5agBW8N@1O{ze);&~8 z2{glDJrCvrxd5H=S{&E5bwirktT(u@MtwF87nY?Ccpi*1mFg7+kvBCiW8iDP#iPbA z@@5G!CN`^3;^A429NWXRio7W5IoH;K+p~s>%{SMX>^q?A)*~@c8Z9SpvV|C@tAbIu zzW~GFZCewrUq_wS910E0a26wyzzdC+R=mR!eV>?`mkcDAsj!AYb$e8p7(>MM??~mg z1GUQ4nMgEUqZZc1>hKeXZ^OEisR>yQwIHB)MN+AqYLFf`RCD?K0<4AM_k)4vgQcBjsq>~aL^oD(pG zY-}VNNmXvz!N96-T%((pw+U5x&y|Pk?xT9gUUO+$9ou8l@mmFp0AysDhL*{qb{&^8 z2OzuC5NhB6kFGZ6eQrE}XR^qj5 zHJai!)6PEXBih|sBkq7k)pn4>p9JQ+&|zjOcsig9W8ySf`>U5tiYe@hEtkx~M2Kv$ zT|9Q@F-hOPCMF;<9=4O*2VS*5(M4<5@tDTXitHunH5cUgP_(jVTd3(JS9l*ywM2Ge zTd_^m{f9!u=&2U5YIR}_>pF!`s9E^{Zz)ZUw}_W|1CX#v_u8zt014}dEIKqz*4&N_ z5#+W)H&R`kC&;h$bz3M|!Aid89dc|eVKCulK;@8ZJ?NP&?VXp}cSM9cq_K>j{Ev)b z0bC*(KmzqWdg1riS+{FU_50wZ-OhffF_iiAOhzxh`4V*2ZNE6M7>Q& zw|M;U>9SEUe}Dh+so8r)HJ2ihE;p)QS|Ik=_(UQT)q~|pmIWIQqEZK_>&wZ&X)Wy^ z?O&IW)NHX~u_ggQ%J|yy)}6L^;+Hn1dANxsm`L%o)t*KA?WX31*c=3h zgB=Su6`H$38YasyL&>qhBiU>rL$3kH?7^u<+C~Td1K@7*%mu1Sg?|~W@{QvomSlNN zl8;3u0K2UT-5SmqpN8XCkE96CWHE;7N3B67UQol{Q0O7S(tL&jKz{VX-7H;QjTNt` zjsPj>&S8{3Nq3fW@yVA&wCUOuV&8mXP4wFc-Y^nG!zZW3t~1Ljpdq;09{ki;troAZ zujWBZL06l?7!P*DC?XX(V!ZZtQoS0%QFkz}$YFxsC zC{^aY$-GIyJQry8H$32W5UciWFK?AJG_j*MQysl zm#M@eYB5*LMrpoQ96+TNh0s|q&Kojh`*ec0K6w;e6caFU6ivKmP2QGvi(HdZa~m5= z$4pTFvpo|EYFDi&BjUHL>molBj(YG2@nA63N44c)x&vR$rFUAZ>@f1u*=lrAQ63pk zI|8mBvSd2PQgEE=@o0A<2y%Su2@UkSlLa4*(u(9VIO&8#Mm9cqZGGy-STPp4FNZ5a z_VH8{?d1<+79yr(7)HWBX!gWYJe_F4wkcL!v@|ceyY9H(HoT4oYkB#Zc7oWBXva*q zPne~V6B+#L-F#BdwD$Qab`B}gLKda;6E9aqm~5gOgf?YkX|4#T##Iy5_y+E1zO8Um zG$@Ra>I(ZLdApS~hmtVH%W#NF2|+&bQV3F@ zyKn|M|Flb8J@~UyH|E&ZN{kc?`Wk%^KI1tol*ik7>?ci9mwQ`9A#>Zf8?oC;Pzgfm z$w&SH9q%yf7Pz*%Nmvra+qX$rQ-%^3O4&sHkQd_2Md0N^=*%YGOVi~XS%Zy~rWf!B z9~!j03zPo<3VMjPOYo7Rua@mh<31SiR#bGN%+o-u8yiV}x$Ekc>uNXE#{Two$G`n= zVJ6W1dY1(_IOx~%Jq72AUjKH=E#K%f2>PmQ@Om36xvQ%%6{RUunH;anM1?lA!cLd! zw9QFxfJ9Waml#=~E|&<6p4MLk#^9ifH8;6jTD;~|S#kDGIa5}i5coosz^FqLc;BMm z@HpV*7i(b}J7VTh-`fi`^#m?eZFOtaJwd6k7o^OefWIA0SObj*dNwat+#I^M5Vx;- z9=bHYQ-vSM66uw&`r7rwrwXW6Cb@UmiYB_rAmohEr-|+VovPl7>)Bo&v?Jb5YkkaQ zM{znQ;)Ut^sP}=9f6ecJnSAj#WD$I5ktSKzn{GhBMo79iQ@%2G_I6k-%ATn$rggM0 zGlu+!eJ3IK9$QYP9YVr*BEmCK%tXsXYCcL(6+aSR%43!3erq#;3wRb9)PGz?iejRO z?KariP%Y^)j$&nNfNLNCZ1rzX2)u@L8EWv6MHvCtc$FTJ9rE-#r^v{w1q*7Zjk%Zm zth8WV)9SJ-sjs{{9ur^Qc6{iWiE!o3cu=WiPjzM0Upj;@f|0z^<{httXK7btF0K3Cwqp zwi^6F!-m`(Fz@xBt;r#|UcP7l<}z1@dS6L@i@1}9@MFqb1E{KJBakY5^Y1|(+{7el zkGd=BcEH!xG^c#Mt(^=Xo`xGdIg|!-Y4Ad%`%upEv5hoNTS<1hnPlx9PKbQ64Bavj zuC{|lYMt?vmX5nMGngT(;VohDTLP@r-I$hYl6_jKG6UpHQc7e?+K8b1-5oiDSt6Eg zNV5gVKe9XSO|(dKogqdLQ^Rnge5P~e?!1S~A^0NDtIKx=V&X$EIDR!3aIB9H0+fuI z40lvkX=aK7BVm%ptIIPpOUA%sG%W0d(sb%t=s!IeY>8b%na-1ztJMN8AHf4%+i8kCH03D_8h}4^|E7NX915Xj*VRr`tGTTqKxj z!c1baS@7K2aY)Q4XT=i50ZUidTab%wWArh!G()&aKUe{MxPP=LCJH}_l^LyUhg^`l zk>+ImFC+gF0QewH8Gpye&dD~sjV+xFAwvZ*_Rr>|nT2&QY=N=kS5Oahj0TBpOFn{? zYhLS7V-sVFwGx4Aqof=&V0BVH2YW($^VGrD4p^u{$Ntf#r=_cE9b}&DKHwj zm?W1*PedrgJ#bQ|SDl|8v|n*b4HB{#^GY%^j|!9(#M$$amI4akxtTsCtQo&ln7Ei> zk*rub-RY3q*qPtEW{D($zLqNsa51Xi0M%%Gb!v|#0}pcii)r;G<-y?%>eTp5e{tO|W3XG~?Zl2>|7D^cfbfPW}<7iDL7 zztTOoS32}g^k1>{(2{)4Tn&^|2EFWK7RJMkck)ilqiH9U$8cVTQ*ZX3QXL8$+MWx` zI}?Iqeiul-=E4U+tX8m%7$Zttzp{-2#>hN!u1cdN#vzu47uT{SWYQ9g8!e1d`TH8A zt`BZJy`y&qLMXWl&_tH1KR7mpx!FW}^n1w*YYv@<78UlwI7^PU#HF*_|4@~oEkUci zu3Ck~dT*xM@$%Hi|7&I(ZPTfir?g-yRKk>)U()(sl)I_9I>^8NQj|EJmK@2}m$ z3Vtr4#XpH~!k&-xyX~K9jE)}`DK>a(ih0!hKPkC*1#3-Qu4@OWFBf6@g#FyW_ zlh$V&t7jlZwcR!sEI#-NcU7XJC!boKdsi{WyIizPIGBS2Lf=wd&!b#MstXGonk&qH zfY58Kn*5_4bN(Q948U4sRuki&bF`}x3{x2pSuw=xSvEhrlkqr^MMlNTeg3xP#g+uc z`~$_!=+_;Xekk!ou_@K=;|r^^*&OU20_EB%g0?U#^!8dGSx{~;Jo$p7os_Biy#`Gz zlT*dZO&m!(>@apxp`~H&`-_@teG0sBYG(R4^J!5>Cxz!qwtxtgUbU}Ij4Jy5f9=Fv zcA$;&Y0S%>fEmMPFcH`Y-!+(|&7$Ya+nCc;7;(NN`uwt(Rz}=dc<`y6$X`O;z=_?t zcC30*y6l=$;;rl1-NWWTT$#IxS@py!ZHSJ2!c)*K>B|>6`oc0%75(0`N~p@JTMzJNC_!*9XXL2$R-9k{EaQ1NMG&P z<)=qThp4#)w|^%e=homOjKhrLGj)6m)y-DR(KIUd6RYMQ^Gt(PgAr>nQxZbN5-S92 zghJMcI+iP$=JJrdSc~tdB37@jl5;94Org{$XuVVsS+46G{3YfWS!ZLVCq85xEy>!Z zk`sUE4yGU`G?^b)9Y8kj-!S<51#c~KbpuKat(VoLeDOi%rDbbma_+kA851~$N?0_zi+ql5x6yUogmK`||YJkf#VrPd@(UAd*d zRyDp~B8G$u|MjwS!PBs30!t0Ka=CGTwDvKx?DOjNJ=>Xc{p~qx6g9hPZfVtKH?0O* z@9&oDSht8z7VZA60KrS2VtOxWT zAl7FuI#pm}rav+!(-WZ;^;LTg)8Z{eBMFG!?A?!335m}7fE#qAKC0uJI7+9P?|)Cm zE)=5B$7A&H=9FEB3dI`BoAA@T`QpnMG2a9}){vf2c7D??HYajq<&t1bVj;;-uY)%N@eS@oXRnGIbl z?>HVG-sSd4#7IQ|(JwH1yYscW(GW8uo9aD2y(&IgZSBxc%P+caW$Tdfia_Kcb2bR- zU<;zOwVpvTG`c-2E;V6turAmwgXnzKEOHL!zQceSbHSNBuE8wjbpfcSc_DOD4f}oyVVlSPW8|8%D__|<;6}o)=+;z1W5nNVz^_*%9^%Zk%$U8nB1K5P%a7{-bN86$2Ruv1nT(Qf zEx=wH>WyXw^TS>G$*9NgbDrFM;5L43)|s;C7FmZ^Xu{2FJebtzvBwO_ibq>GcOSki z?y_t%CXnBP)$T#Ef~^l+$%M(Xt<&Li?69Q3Ja0EgkCdtn z$@I+5vP?m{&%dN#;C$~$7}R*;mVK#FahgpKrs=W`kw^m|7-$- zO8g&5V3ZO6lfe8Zf%#t`FuOnrs5RV=PWI2|*BET3tlM5DsQ(X=FtVR|>i?32x%*#F z!qggwZWF2=TR+e4In}H`#E=+}KREmcCSedq0|3ZlUl(BhJqfe>I`JX(`t>LO>|ARzPms~}i}CR^L*m{x4|j<~8QH8QIBskU-OktfJNJR^N~ znkWX$O%N$2SWZ<993H+t6E_E}Wg-T*yx0~VcQLftf)_79tOd2j3E``h7JciG{By%` z>pwrLMHP7xiY_!(YmBG-ULf}xsJ657 zoW6a7r*>SZ<)gClViGwJv8}BBFxo{aMqgROH-mzYhX^0_IZAj4^;|L+K zIq=OI-o_eRg-l!AVkC(|i(%n>13rmX)8$yrss)O}L500)pN2V23uSA0k99G)LRhGc z8YN82WT$MxGXW}*ff6^}&hH5g$UqGvd8W60p);i)IgjT-BkX7d$Y2fKR-`~}ubBas z?uXp4rPLQsLn`d;M&Mn`sF6+PA)BJ&Hh~)x{rLMeePtJO6z-+jJu9#i+wh(`K?m4K ze1$1^o(pU|806X^jL^EpbE6jo`u^>ldCocs_TjX4_eQu-u0dTm%N1Ra^@XhTy8xsm>L0oAcW zDY-N{MR~!hSZJ*m*RAaI%E%BH4JxV2B<<@w*g^ZR zqEu~0EBp+V!7Z%PqV6M~i}+)-Rc?EtpU<$p3Ce}QD!d+*ZSdIAk^lxrEA*0RsMIXG zUxW@;;5reVcClPX_cGCHrWnAk*VJS+lMh3a@2|=#xO7}o+<%RV{w98vh-*cuJGxp2 z8ADr~6`cAtg%Bme5LTds6-mYhg(q^atR!bfhJ%B2+mbG?j#4J$DJ-I1YfeO1Xm`|3 zW#feF27X)vZ+oa_a{cqP>|ZW2pE-DbrAX=cYL5PP6F?|U?-zSYauwy7DT1IzzmB$N z#F*n<{{4$hMw045H(zb@S1@UD>@Jwg%UYx>APt17B$Fz+F6EMf7FipSW>IxgLSUY1qk%3pm=mEY`#lVI z@`=Rt;Sx>yN7nA!BGsediKPG!g2OOevLs5XRFXFug>R34wC=EJBVl0#)&t6g2lx!} z^EhH=o2T=`@{_61`C^Z1T!OHU)P$F%A)2nJ~n|DqtfNVKKD zlAx9uq3vfqid#-`KH`NSq3*4m8nQhA2~-L6mwPYi!rVsW(LhhczYD)Dm*4H@X(#>N zJ6}a+aX8XR8uWtl3heuY$BS(mVj<&#sC6(n*BGldgMvj;?yz8KPufS9b~Qy99IKJ+ zXI;}_fGV7f|5dgL@Pbr&lb*w63PZuLkZhoM)M8CvT^X#4&FrbA#08z(z7NqN*H^7N zZU>VEf~4Vi-!4Hvgva^?P^f%_RP#2WEIA#f)r8x6RxiVhphg&FFCm_if4F2N{MAyg zkNNz`pVdD|2I}(93k++wp^P6E1s$fNd$ztMZMJ807}Ks(i$lZe=ZuHSB1AU27}L|h%lu9lUnQnLv66d`Y3q3Ph#PLQG`50ZaGS4!mKp( zFpP{%x!jV)!qcGmLAouFGEdqT5POg@X)?7w=+{lZ;54~WuZ2v$$BGYM&O9}^nnf7a%Q) zpRGituB#?vpK63xKHaqh@3)nb)>gSlnmlEVj$rzj~T6`x$< zV)({o{_sb0zZx>z+mb;n+M)==M(6Ynm0?$R^jkBME_#|v;vt`SJL~=XYLgumHOB0s z{PRId>Lhnm8}_h$HkIFcQwc;cRk92k4J~Y#b=U&fLSBgm8DsJ~J;krl;r59;?I`X$ z&p~z&utsAN8?=7MBALJ8v4&sk*h~5mlVQ}$g=VGmxu63jo$_~9si3R(g<;N05c(zeehb%oi+Bunx8r3)1uuw8o zG~FnAF%-d_8dh-F54x;Pw9?BHB}t$&N7CCdB#!Y-HDdDlDb(#7o$iL&|?cFR_j`YhB<~ znde?>83gfWpJcu}uC8H{%q@@F0HIi`U^$3AXm?bEX}7iOqOMA4!2PAIz;r3B?;^TU zSdU|Hz8GTw>Cd)p7^(7zijMRvLs_rQlEtU{3M{8+^g{YfZGL)$tH}u#n}UDJY-~sl zt6&y{aU^jX?%-UoXn>ur{5|{e5{Ax2GL0=mL7Gz1b5{<4Os=TUV^gv;b zgL#yX!p!k*u@mES3&P~nO*yO)Q>Kie1(dALDe!Jge(?~ zJ=Y!yo|hl?C@k#z$&Ev`v#2`3!t1NvN5@ZbH7o;0Qn$=-q@|hl5DLmS-OVU#DjYEJ zZrL6%PgB#)EGTklJ8WE?wbJ4IC=}j%G855CrIUG6ES6~-p7gbQlBkl|K{nT~1XYEJ z2p<73;OWzW&(UlQJ)#a;DJcY?SDXc@nS@5UV&xsK2WaJB7<-p)7x1lU9;HYe9-x8? zOrlm@Dztq3;P3A;6dDV#5GM)}CzOtE@NHzYhm$s!%q1fy|AETV8nz-$S03mhQll)t zdCyy{>^*TWGvyeoQlo=r@Ku|^|3_^`HYo4dpy;ngWJOZ5@HT1 zRn?u@WSJg$MbOr|W2FX^T`Rz_4Pr`l_#3|JEY!i`5nJOR&mBTsS!Adh^_qH>176Vk z@Md+AU#KQR4sPw}SVTfxzugI9W3x2YYhNdWz~Tt*I!HAorR>aIs0OaJjs5VPND@27 zGdC8=cSTS$DEkdZ(?+`i0h@{ci@JM$j`M%uejhuHjmEZZvuSMGcGB3`q(S4Pv5m&I zZKtuVJ=@Rs%$y(3ulM~A?Ce~#v$K1>U)ST=$mGu@Xm0ch^--BOQs6E2f-U1>mL_)uN_GB(c%C?+#&Lr!u1J%m;4LQX`$>N1ZM z&si+lkgt>ZRV#RX+yLrtjuU)!+)<=rSROtOZw);{Gxjypz8PeKU%Gn2;UOcgYO!_B zwf&bAyFMd%g7YjT_6y1>@>^AvU6aHow6Es6OvjYt-yS1WqgV$!_qEld4i}`fgbofM zDkPdmdGeqOv3_y;+E73_J7qPraZxK+(@ycu(&a(b{b7_Y`|=Tr1-)6{9tVcE`VxA$ zcFReR5t6K8;qiX1;=~8TY9pO`0#j$Xh3*Yf87?E`??`)nhiEyHEBR?hxunN$*Mhj~7S<-Ez7_EB_O z?5q9?Nrl`6(x@-^$8OHLgjqo`H-F|-hDjrhwu=q%AH*P5b{O85+rJ{?6;rSxSFg5T0B$0=^QJB z{@d1AviaDK%3}$BHy~GR{kN?dMdSOa!XQn7l&R8Aunp7M-t7<7nfDauKq7htyQivhB`Qy9?kPFJde&1!FT?HlJC!DcoW3{36upBz4d)Z8ffN5g7{NW$?m9m5XkJ znZ`y&5X^YV`g^C6iC)BrYsv`$F^0f~rre2`=E*933jn&t)eJTQZZqfgs=CP3+;~}h z@}x)_G2k1p>I~u#FsNZ~$G%W4P#LR4<0R$q_vG@4OW1Z-HXmCS-;uuQ0@rkGj<|J4 zBQ}yILT@PPa+7RivA0!hQ#!8)pOF{W0rnCB`P00*bu_gXA@~Zf>iDPhFyW5HA4hEI zS$*0bm*ALc(Jqr_5|w$ zt)(6fRJ8p?KsDXlS0Q6(XZt@fdtQl0^0N@MbawD(c6}C`V~#KyMeuL_R9ov(>b-nH zL|_Wq&8i2;;%Cw63-fJS>f?a2-tuAor2St8u!&~U518|@?3P3Xq0guar+qWz5wTQy z9x$nO=*Qp`fTN-J0^uOr4X@PUCd@sC!dwY-FF=6#AEXQFFqhTuki)zPd=m5zDODn=}8gd&p-$*ehZA zkCn-5G^$%+Y{Pjb!d%>-&rx z0dpRN7TkdqP@ig8`{qF-=%OiZxVwL3SD4Ec;Wy`iTs*WfmBQcGu=AInGUl zzf;29`+v*M<(4Lk%C}N#LyhgA!-E(_P~UHflto||dPoV9tuL#IR_ZgImUQl&6z+N0 z7eU!*%;1?E+VJ>e!i#rBkU6ykmSYj`_XN%am#_l&6IlIaNk(4 z*b&ILf2Uf@Dudo5z>ahuCaF(7*MZGW?vQPo;0hYls1m3+_Q$%_-nP>5aPQYthg~$$ z(W6~xnB>0(JxiKXDd^-8IsGz{x@FUvR%3wWL7<6hFi9;HByIH^WaX+m(K(w6d_%?) z7++V)!sg^%4G@cw>qPG1q)wM4@|iBKW4Zq`WuiSR9KS?P7#(54DeJx5u`k@SftqNy zGq`Zld++^;3c=9Yq{gu(pgm2BSNbGYd&OLX?1i<`eIM>d2ZD2(aNctyQZU%1qkQ!R zT_L)`wyts4^-)1u)X@rE9O~9)E^-bsgqVhjL4A6JKk2DVR!74IlxweislBZpAf&3d zUg0|h^YHf?e&ux0e&e*wHiTq>($n5%(hn!n*jv8YX}(av&NjE!Ea;Pltd59(X*26R%}~x zAa2f{lJp;JH6Gzs#W6=)(;nPiM;LSXZF~)ln?TeUsy(i;uA1;E$WhgpA6Z<(oj^$} zLg!~-ILLJ;3K+SZ3nsKEkA_{dyErJR{myf|A^-cY&#zazq)pSJib#g>w~$Nc23TY( z0c?9^ZFd-|zHz#$Uy7||h7jD?7l~5{=G*7dy8Cu0)y%oyzaoJn7tho?j+tmVY13L7 zA1AEUu~yD?Va*l3i%{<~+;X$I0%&MdjMDK(9m4!xOfoe)Z}=d;YEWbpyDt>D4VS~ww`tQO+epf?o3UvT8m!sE-y0L>Jy*H81&PQm@T-~@Qf~i8O ziyeIKzpS_UT$s?^mfc_U6nKr7ekUXLPYc}|P*X~pos8z+u(X@CiVc-@S4D_6+%FLS#3{rJRG zHeIRj`x*OWtz8YEuCB=IMkbJizY9d`A z-MMjRjV-~jX`LI7%y#^E{lj=(z|O`>*W-`v&CjN&t0hLyrPgcuiZp}NRn9--xb_m2 zqSBHlq#>Lzr4zHR1A=AfFz;sWzirXnigO(){s+pS!B4ud(3fQY17*63i8nlSQ4ZoR z+{vW>?P(G+6;pkC7u;Lm?OF)F*Fr^5{B*YIPw5hC%2`_@5Yh7TQ%Ei}$@=;=B9mTS zH500f6#npSpF=wij=tWZ_0_0R`jI!^oYrFau6on%$u9bEn>AP7QeJldQ)!*EhTU|^ zllvEY2^p|**Sl5+*>)m}on<+bzy0L#yzD^FPG`wgnKMTvN@j%)I|+HZW$P$oSZ;e6 zY>!gB1Mj-l+pE0%KHoxcBN3hynjhl6$kBp(ilYBJWuNI{p&tIqb+N1MO}#*6kX=vk zRk?*MjCtBQ-##3ZjzpV2ja_bb{?oPo5}s<=>2x&b+pFc(>)G(#-P|e-!B#0L3eSm# z?|yiY?$TEVB9C!OZmLOwfXQz3?1C5#(*`50>2c1IA|j9|bHsp~Z17K%8D1H>+&ht8r0Cbp0r4wI3}8T`o9^1lEiPQ zp4I@e?nX3 zFw9=-iR(iMPRHow>jenm8%rlbAwFN^!&B*+wpt8w>-+Ann*fYeUTnF z+Rqdcltp%N;1E4Scb})l%06BBQeoc}kA~(6jfdBFCa{a|xIwhGaV8&*1Fa97B)`X4H zGoMT;JT5&pnX!}%DWm_T!Lkx?g)3j_S4h>t`AS~Kf+{WTsPN6IItw!}0QlJx5_>&Y zC8g|8Xs)SrmM{I;{qo`Ht&nGa6A05Uy8roP_V?!UxWxSBn}bRB_CFEoHs(=MhOD;D(7Bgvv$_gpGj%F~a->yIO{q(N>Hz~dm`x`WZD{PBW>LdnNu|Hyaj zr_-79YZy|AI?*egU2J*DvPWq$)eHO7>o1?S(i+4ElAIeotu4oc=9BM~ukQkkZ#dtM z)(#j|=U0$J#)_C7FsEzKPWIL)xGpg=KuJWwZ438~NpF|}q3VFM>C#Z6@=MRZc3lSz zpS*X0(cb#0{zUU#G)D_FvsXjpsfH`ML~VH+f3bmJ%+a~!CN{8bB)Eoh$8~qz&VOjbZ2^r-u*Hi7d?g*yZ8 zPM!#l)`X#@lsMA_Zp4_WG8;43O7%S>BZZhtH zo4+{h?h#)~0+a5pF%*lssT1?veMm81elD3ESJkr`Y;u-2+KtoMp-e*1m8DHBk@>3p zL_O&()C}_W#hJo?<8`n4{YFd6BWLuUstitqaWjT|yq`9c5P; z%>}UCB*Ra3>EsjCjD3tWXt3G-Ff~`fz0e^BzrbdQj{d3JK>x+FBf8oB@KlI+G-lw; zA}HM0RvYn5?lNB8*}X4Cb|z;F9fTMjbz{Lu{Ngl!{<>#?qrr*11^fgL;I0Ldnd(J< z>PG`N{okLR=Re=vSXx_vTZ30VkA`nZ{WNAynT7}MJ~-FgsTgl;1O;c*z`w9SpN{8l ztmB8Jzk3zp2D+HrFnf#aNP%EI6%pm@Qix2yri9VGZr2lr#%5&ScHL`}@f)eHsK!}; zwm{Hxln3huVR@)|VrUu5lYGJg;vDIKRZ0IQHBa}}?7;2h-p+SV?rujiAnmgu;;g4H z!y9(uf0ZS$Q)3YQGq)@?>CWwo2vTLn^d|`6Q%hU1SA0HFw{*jMnp8*LNX>IxJdp!! z4~q>YPVi@1I1_CRXz!;H^jxcMxa;`#YZ*kVR>PKtD^vL9mM2kdw>X=Gy7FE&5kRO+ zT7odv{8F%oN||rAo_;7ul5|i5i?KG*Kx` zyU?M*RGUjnyLuOQB9H~=;YB{L4Pvzp_=&ox2hv7WdmPQ;EM)d?xBIM6QtJVFjF`Wi)W#Wx zGYn08L1T5B6d*vS492_^XIgGq61!EDV~_TP-9mg&&>x&>g0IEHl3=cyADl!0?Wn*1&EdLb@KHC z9qwem<3B?L=j@Hw%HOVgn0KM7KWk9lCS91#uLyfFD&sh$C=5$T(f^EdSQu16_8vCB zJ|li{ME;XSwujnus>-_N2TUSON@y8@e55_Zt=C>U`PSGQuq>Hp`-SqTapr{0kg#h2 zGb*uzRXAn@d_aQQ1tSSE`R5}}%1>+|KJ)#8xx)+t87L-du^*E7h^SwN`-j%67dq;tK~*b#>*q|#>RO2P_q0gn%7Z&dWv7>)rri6B+xEQUO(5DTTCp>Zo5NcJM?_IkC* z^{y$Ho@E~cEDgMk@mp7OlQZqT0ZSwjh<-N56Npwi@xG=hFWI{)ked#c22vjb0*lmj zCV0F*Zb2_pPxm>s#}_#lI&05hfE9>RfJlr3zy5iY=^M6P{sz2W{2XtoDX?~lVF9i(IwRMy&N}T*v zc|+Cn6({^BwI@d_-R{1N3)y7r;N??rD4$a-7AVlaxBxIt3*m8+wvLJL^7LTGf(I07w7rtlvrNd| zvS+fkgk<%~LSEJGTMj!H-t7y6Z*=g-7X1NY282mJF*0t%P6QQQT&jW^A^%mDXutMS z17^L*ECvq<{_<%zPyHZ*?ycDaxiax-(*hRHfYi(#8d=+N^q?5WVaZgiv~;rU!{E(Z zl^NR2_(-PsSqnvWL>qaX(hb1-<*u{ zI_mogqo7d@@0ANl80B86nKLph1FwpZA~iiyl`xv{y&@f5q^lwc4lSixzG@_rHd(t? zJpuP$yf+fCepEBOSnrcZ2Br&Ky2F zl^pmb_;-dzlUMdp)w1qEzbF9Jy@(WcuCoXAP%2e6IvsS$h#TsU))B6n^s!|PKYlYa z8)SM}^O-YHV`XCFF!WNAg%1>%0(aaoq298-Imcy~KVW;`%NQhC#Qb?YMx!7NGybX> ztK8^AZ-4De2;WKGG7{T>z-e}_I=Y9isW@8htfB~jeIPAKD$E;=`2n5lW}vPtB1oe_ zQ+lQ}kiv9~BIIPXH3U%Mc6tXfD{@AFEuGdtc}gn_p^^ zew}vb#5I87UZp$?buk#AtyB=8zin%c#IH=Sw3y3^Jqg2AavnkmPh<@y8aS1Xc_n~W zEFU&An7BWljL|M)UhGmYcNi3jPj_tPA$R0KOOsV!gkLqnVZb4$!%v^{lWMS2+KZjl zgG2h0oM}Q-#94eZmHwv($Uv9eABlL_NH?YZCCEFSs7!oz-0}tr)>EK@B!+1rNSf-J z%1W%(o~M$VqPI;io?17K$qr3pc-DeKN-p7_FT-ofx%Tkv#D>BP;6`>kf)6puwNtHm4;a|rTfW_79&7x)}^OGl6s&0Q|MQgNgrB~sIu zOlFm>xVYL4M-9D16-19`AH^lnb8d8ADz!tW++ipO{soslBy#Y%LBS>U`pz=5P*o~i z>4Fr2Y<5z(%;>|c+Kl2~?3`;hwnuu(@m&0812;ohJeFw#+wzz}<7}Dj(D61I)QxgH z#2CzGoz?t*(9PhgKCv zskB`{ou=ciP`!T!R>`_5-OAwOI9X1RDIm`)Gu4_y(TAye#xqdcOTX^GiL>*&Uegsl z=3~9F&7KjPCr&+cwRF^F3k7%ufH(6rI1-*NCNxv9DhwrR=Ld_#HP#rgwRwXwsufmJdp`-Pp~FC#H4e2EVR%x1_P?LQ`wHC0XiLH zX}{;#V_aTLL!m#LOCw}W{2YHI6#pC7U>zd$2x(}3b-9&8Q2;l{LaRq{U8*Zb@0&%! z6&<_{4I0$M|0D8;I63)?)gru}fwrQR#(||%en>JhR0Sic{h$I|ByG!qieAVyDHF9f z!tV3<=$zaS9G9X3GKt0}ds`*Y(feO!mkFJLRo^9*LLNR)b7@dtsF;!vad2frbh59W zE9qYRr(nZVLc7NWSXeNAzZ)}krOA@gKpTx^iS`(^qQ0DA<)~ZMkWGq&SYw|>(8+W? zA67(^3gW2BfywrvWPt?X%hvb*88fLvEbFCnU1Ze%PNcTaBVxm7UJF?3;$wfySs9n$ zo?(dfGhCE1yb`|MO8U$NasS9ZBSZ;dHY5u`t8lA9gZgIwMlDn7nbfSeRr7?5w zQ^}NJvTonx@l#Ey`GKWY0N)1J+Xeq328-I}T=O3?)5NpiT9b(^R;k0zBzN}PO^QKf zQeN;MGP6j-KwRy9J6O%;$G&wLneu=J`K2!On`Wy8yiRAecsBLuc~pn8tRO*mk+*!C zR$nq1K%v8gm9F#12A&>IIU7_QPoHZ;Wn}n(pxoCM43QTy@N^K8?ot5_Y-&=}pf!Tw zze%fVQ?xWy2Z|@vPsjOwo0hj2c}hl3A`N9`972xs ziI&582|G&8`Zu&iD@b8p$cRqMJG zYv43rLIvxikvtr|hzky<&hMyo+Vr*~pg%_qGv(v+`Q6zrD`0%{l4Q=*KePKVua#IR z;W&_;eiTd!cB+1h#e<=Gw$O;nYt=pyg<-FwmVuDpK%)xkF4-}0TCnbCX}IDEq6BM2 z`+6xEj3#=P>$DQ(Oxf>p&LPkl>3yTe#CYoXLaVlQ>-W8Zz*NmPRYJ1}Y+s5UE4cmp zPKo#OaR3&XdF?C{zWcX%N11{st~K7Idlc4G2QWoZJMu!Rqh%V}h}4BJwYMgEb@EEP z6;?FfK3nU)A!{Zx57iawMYUKeEBXk)A1>qb!2f)`p$wE%xL65u|$}Jqh@^zh)W0IuM zwSzoLlXEy1n5}HoT!VnII*bm!uCv=W3gkh-cxbq6)r@&|vvm)QOnr0mylZF#o z$a(5xOK3^#NsO@;nQ~2HBg~s0b;7pRog4TXD{2=iO?VS|?fi6|>8J5Z+%RbHMUPD4 zOr?$@E=5O@5EkDA#MC(5v2lnE#kKby^bLbYGM=YUc_(`gX*)Wo#nusTeU;j# zAD(#B1YQUIl|Bs2j=kw@ig%%ouBB!eFuH3-f88>ynVpFNxzPwXAsq?NH0v&+%-Z4g z!dd@0Gt=dA3?*EBx1`iKT9hDX2Aq+kYNTUTm7Fv(WIt=NiPu<8-gzoY1DrB)(Vkb% z{g}Fv<~ViJ9%^J{ike|LwMP3FZdNQ?Lw}l)zzh|nqW>V=_-?J)PFU^)D7&O?*$5R@ zbB;Ykwy}|PzyGwhr;(~1jgVL%r-d~ti4K*pRdNW&Rgbz7sukncz}WgpB2iJ{%GE9A z{4D)5!t@7oKH{T!1|@41db%k7a^+qE89J17EZ#r`xa&5=(V|*&lxh%+pU4+5sJ=>S zd8X)3q@Aa?ZIgH4*?Xl)RyV)=o$@*}SYv}&>+q7}FW~e%wQ}aCa6&}C?>kWcon#?0 z)&mC>eLllt@RwLz(cmN0%2{M6s=|OS)D9O8UF~rH*W6^|#s(bZSIApPJgIhZ&~opp z1o|slWQApCiv0Hu?03}jRC4HVw`mQ1fkvB4K9{B(Jiv(WlAFUTM2wn|Qa$K8sjs;q zR=CK)17HqPR@~4i5d$|)P7|;4{)ABZw`y>e7=$nyacBJ^=;xQSc{yozq0(rl0wIXv zS4tdK?1zg_-R`f|Z9|~HJ-f-f`xfuZyIcFdz;4jr9u7IAzc}&FM?y0>fkr`Z>!0h~ zAD~~O){Ib$qzjP{%-UJ-#;3s=wD_0n87v|)Hk!9Ia0+**Mk<@2+LF7@LMnfh7};qI zsJ6t3(&Qh-`aGN_={H3usyA~tx`cT190`(Us7lUHwzjtRcekFlQ9MD57)#V)Q#smA zq8Ur&z0MNTN4r-0wbM?#$8DrsC{e`hC?~y9(=!DrM@H|m0_ic^LU>I@I2R^fu?V*4 z!^lu$b{o(A^*qm~mV`W<`>t7JisQ|lFA#3q7lKe39rNvQjnCDkcweCf=Wsr%EWG21 zU?|$xe(J(8am#^XOLlZMdrNMh*b-~32h42VZ>cb&i>G9pts0BAMoBCzk?`|BNcj9Y zm|2HYy(KtPk$%r`_P@%A0N-SA`TJRp%hU{JB=PXHgBYaEcvigY$qh&L)lyX)r()UTAs*@v&CM5N5Z zLr;xd2yA|A9a%C8&@Pb4?%NkN6PYS#uR-E$M5Cb{CXQ>(b?QR3@!@KgzQJK?{icNN zzv$7=@9mxdrIy&VM6U&ZV`bkSr$cTiJWcM={Go@|QgQLhp}C2FMDm< zYkN0jWetfXa{ad1d6hr3Y|1uHNkX1-Zjh}~PYt^DYF%oyxRAv4Pnkgsp~F=jwVvQ} zDv=UI28Z+=^YR`WX_P@V%ZS(2@-;J(s-WvMvaex)M3(BKAF}NVtN3I)RQTgp#5roI z4YT^)>8os1X$$5X3sbqg4kZaAZdlWAut0wjp=B%eN{xUo9%yPf^a%O`1(T;72V6fe zL?P|s{FYJC(7;2s#)J~B^QLK?NA5?)gL%Vw4`UI-MCz(Gr`64Y{nQLh)ry`fgeY@< zv^pe`VT{JviYt}6FOxmx9LC+#1HS8}bXwH4kUwb0J)%2|t@P27efV%HY>~Ob4)Uvk19<=P`?Jt>f`zxB+iD8PxY{AKnVVznA=DUu^TF=3#I(-?ns7$6?2jnCm5hfDNdrD zZ}X1LDjPJ|+Qr5q%k`i>E9-UMM#I$l0^1rucpQ|efl;E(J507h04gkXyy#m+o;MYR z;B>SBis5b>dd3$9xmO>Zr?If6BW?9xQ#$EW2I^`I| zg+k*J#Bn2AY5oSITrbDy<`nM@4JHMo=4do#JSkiH1t4(2$U>xBh6i=kmYzvuESp#i zRJooze@RR7w;?~ZPX~O_Al8KbYw^C=RNJXxPFb)i$Dk&R+Fd#kDvQ_%$%HW>V*qpZ zEBX*j8l2KuV*S{~m>&H0}L1+mI`&BuDmwzrWKJCe9pW;26kp@=Bw z#$u9Ygi-Fs;b9USfH#o^u^UAn7!aOe5Y8E+N{kP=E0Yz7n{7EO3hJ(6`E9YqT-#>!p|z$`7%>8H`+QbhmF0qtAL6ABg@ z%r>S$5}j3F+4?iV9;^&$KN)L}i9<2ax=w%ByF{6nmK*y^CJ0P~%3+no^3w@8K=hBA;J}$Si#$bBXYG(~Tr6W0)WD zCrfO2!dpl!z`J)07yE;=Dn(m)!F9|AzfFfAL#Pq0K<_w&G+!wT2^fWDvS}|lsUO!2qW53W&vZ7 zXiy_}1{q|_NYbl>;RdegYfR>5c%SiGcYwAq+c%Mh z1HZ45W-`;qWZC|#(#*+0!$8{@UPZS5t1DfEX46kheOnTEZS;F5x;o3}#@IA4q5=y| zNmA^k4YgqI~$(WILUEBg8n)c{|{a<%TOJu?CC$ zQ@7PuGu`eN(Ib=8xFT%(m{F6=PW@OYyVH?zm{Zx^=9h)_hf zA6N2Z%&F(P+5B@jg^v?p{xCPr)-L6V_d70<^SG2o(5!X{EQQx6zLi?N??-N>gBxS; z)csvt;cps4I2F)rUHgm0VWX>;+~~^kLdd$qSx`q^r_#ZQ6|euHbHk_$bNZi4uAmDK zafH;J#85UR`_?(l(j`@0Ur0E_gaqp(wb;A;eJ9r<8=K@FZi;-rz$~f?G-JsnWX-R! zk0V0VjSPe5URzC>7deV`@BBxuCw8>f8{5|HNA4CAFPBuVM`{PToB}8s6~=MDTw#uU zTrIgba|BbeVid=HF3X14>CoVH2gT}lUzXPO?zB$c@I^YzBDo=nxlokVQ-l^OBM8_? z3yq|Iblp#$5B)OJDwpu8%=jq}NwDSSz-W*f!I_Axb*;b1Fq(Isq;E+G;$>)B5W_AS z*zbwE#`#R?)IYTAEDV9?`hg!_0S0^o=KTbm2jJA3HjW!CPUEj3Mg0zK2SbJ4{WUL| z1*&edHg?!ZXWnf`jg6+|RkK;prgQQ*+Ae>2!vER@U?32HFUpkmsI;h;l{@mHxI~CQ z0J(~LrawO}KGJeR7QgSr2%9tg3oIcgof`~)F8?UJ7LqVH^+3|!1AZKwBo}`;Iyzn- zY<+s-2x}$iaM$&Cpfk=qjxBs5W^s$Sn)sFBP^KYI%?se6(f8W1m~$#xDN2Kr9zS=~ zk0qckZ1qrYradUo8&}KZ9GQ3~5AXTHwQ}g#-P#p4USQ}H2UbOEk4hi(i!0>ZU;jpY zLY>Bdc4st?r=oG|EmswSV7Z8hV4uoj?!6EC6tcAv%&Z5SOs2%ou`QL)`|Ay$|2bY0 z;V2<2j%CT&$oE7lVCw!{H}16)b5MN$FRujb8xDQ@_(V?KH=GlYy=!p%Ll+ZER9|PM zUDX8L2?u9=Rn}yE#O`#SRDnY8rU|Kax^XEP9g_ew7D;+(^~sQwxs zwJ|3Hx>gw(u=TX0srvvd&L_eRb*!|In=q#hzIHev5_JSv6RS1&Dn2|D;;>{aqC5{tlaqch@=OLXnup4_QPy zS@>?8FusmgV{Avfr`mH&mzE>+tN0}nsgZvAX8=B+lUms$7b-NCH;1)F4Rn-oE1xro z5(A;9je2W-L}bb{KK6P*>cg|S<~P=MoE)Jb6&FxSTw1%d2kzxle_;6{)22P|;{sMT zZu}U_c}rv(m6EU!+9DN5fd#X7d>jXlFqx9PH+{AJ)hiLr;LqamXk6p@n6=>j2(ZN& zsEdtZrTw1U>on5}6kJ_zPXM}8cRRWQf18hjbeS73^(qZ8n*gmIT}2rnoMrJEpqn7#}2QHK+w;qkjnmjNnR|dk(o2E+oM1F zoBec)USo7sQRv;aCS2xdh*dzed$M&TBIZw{&IC>+n}|( z1#iQXY`5^I;ApP!N+RzWSGFgZ+R1eJ`AbM#NWbCd({_BIpkKqu!j<+#+ySh|UBUO4 zKg!94=ntPR&jGGzqk9C2GH-h_UR&6L>Te#X=+uIB&Ub?5a;*S)=_(GSUbtY7L7nb1;UiX0>FE80Xt^2fd4PmFE}J4|F$PGag)| zjD0tGPoHD#jgIZ;xW0Avn&;~0+T6kvNwUI{KHi7QAzM4{WZ`W(+qZX*b@ux-sYd)S z*i#SoD22&@3Q8%^useeFpoLDfH3&I120iX}+$>sX7b6pVa7nnjGNorvR$8`c3ZJ|H{#s$BwmPFqNopusq z+aEd`dhg1%(YQ8yUp=qzJ9%8U?(L68ppNOnV(OySq?h6=@KOPv>ox3sUU<~|IjVSW z8)IaKuM;)ePtKO@USk!v<`Q3Br`a5q0e2e zge0-|h_Ofoh9&kQvURd^r(Yqb_R{G>=SuhF>^xwoLxM{~GIONK0WD6p%^*96UdmUZ zM!V_iv#y>d)`KzOhX*3Iq~uuDDpmx)f#muh7x%9T2ah2a(pjYr0<~)$B+NkkC(MMJz9ZO`keirM16kYnNB>TbpEU#v z_#Di1>G1VN_7l_Q;5}o>YU|7c(m^;gkHM4 z!^IR=P9W^4{hIf%8RB%97I9MCpJ&64k#^q(>$n`EX62{Dw$gq8IiHl7>~-ck8u=%x zgHCpEs9aBbG+`Ztny|e0WR?Pee&aN1 z)k#{Xp-bUrE1|*+fbA6Z&P{mEdDz{#x%q$mA_npQcM%i$zeNo4|GS9seR|C0fAL5D z?2gy(wlwVFd-1+!KcsNG_*`d<5f0A{!yo3Dlz9&51BI4Ue^5g2&#Z(R7&$y^aGC*k znwAP|q$<9veI+Csi5=$CEHbMzS09L_4Caqu+Sj{&YRLgEJ_T|ZuW|^E$3$1H4%L4E z&06q)uC3gJ8&|;g=Z6vSDq9U>3;rwvHM*>+Ii&jfR1TpM%UJ24iH&jAQBY^8uRb>N zBhDwALwKDGYuqZ^fCWreciK7!N-(K`U7@U9MGTcPm4xfH+fgcA^UuhxyeK*?9>tNV z3TmeSpHO*BG_>F?#9VP=pue+3QsbVeTk6vKwIPLYq!O@mEgV=5o7`Z3^YuYF0N|Jz ztz^x>Lkw0QB8}F~5F$NGD~LiY`Jnc$uXdt+eGG&Wysc@Ji@~F=G3?qCRnr*U*4Ben z$zb^tH(0TECF<6@PiDNn?dMu^t?Li`+O-z=@~FFhiQyztKqHS{gvWi zU^60JrAkl!GxR7byHMw+HKiewi&!@1u2+Y&=r#VKijoiqe{-T%+woa+-9>|JHzty! zu5;Igg?eUQbp>vfb|%-cy@KlnGMlkAG+1Xu)&U!3>oLVz5h=5^pmm(M#N#OJbko#2 za)+Q}+8-}NO7Km)73Gl_@gx^9b_1Hu5?Q5c&O{F*R8k7sL-J<|HhkrQ%5=YdJ?il# zyxQL+&*~d$@RK*UzlM>7=UgxdDXkUN$((L3T1V>l&{Yc?^lEj55MB=)2V)N=2x_2Y z3h2yRw5%{j#^^#jRH>-Jlj3k1GD_f83qwq&U}C1o_NhJlPI-7|-AhbMMz z?3s#TQ>hk^7b4}`Ny7Z;5LcwaBX_6fL3@}2KTz*QbjeCWJ6FRWkQIf}fm%zX@@6>l8bL7P7IWEit`kYPHA7l$%)m#p#Qw{a8LzF7*H@h9v9FQCWV_skH@aM+*C>MjSTBV_Ki!6b7RDRA*Wtmton4*y7d_VdoO2e@TIUs>N-}3>vWP`({*YeeoXps58%4 zv_9M@OXgXyJEtkN%5^UACNSjd=x^z(=!`gGL4*K`n(Xo46N2;y`)$=Q{o^}S?f-I1 zF+5=@QI?WII5;XH4>uDE8Of}jsQt84TD_=jQnevkA66=Xzw=}t(kEhm5qEmCR@@h@ zwa6`2;f|o)A>&vcAXD$rm6h|65}XGgnn*}Os%A`ErB2Fbv1T9!iF&%IR{m}r;@HAd z#d*$^GwDB9W zirFwm&_`-5c0lH&4m9(iwydP;$Qk^r;|TWC9)NBgACo1{bqhbgA!R3jPuvN0SquYt z>h8k;DHV0s%-~Y@{)3Z2XWTtV=Px56RvUP|LN(F!Z66$91btqI;T3*z$l{6q1(7+D z7KF_Nl*;t68_@s5X4~RQIX$V$0>4xl>}D#-R3P8k0&~si2Hi>#{I3 z<~dx8rF88lv(83mEiAKnwpt>{YYh*3qmI~D*Z_ANj?Icjw%2#g-wSNUCDn?9J{B@drO^MG>S zIVHHCw~eeuXx91{s|%JMpz9g!1=#h*Oyg8Q8IR#2xwEp=W{@)BGe*(_F4w|93SxO6 zWR(_-@!HfG)2aBmO-RPhBLxrR2k5=#C?!ohFvY(82emrH| zkZx9P*mLuhgKl35&>moSVnD}qoM($GmYtTwdc9;utf5aOtS;7!u6Xnh>oed|P$X11 zx)ggHe4v^ANTo31WV2M&M`aZ?2RJp9G+mRT5u%2}b&0)vT~2MMK;sUg|JPi4NknB~ zC4XzJiKzomk~J1}a&|tGMJL3JgoEQ!r{dLraqWJQ5`rG_sA@ z8<%cb$JZ|Oh>%Jv9mRTdiD4IeG@jzqP#5FL04Fci%3-N} zmSe|}SvGZ<1Fzx>8mF&-pNxJ-Vsdk;4>D?^EUZe02WLG@6^OP)6&g6CP$ne|brLau?B!=@>{Jd6Gl1!w=3IofBwnoVw3#$z& zZ?xWddxy-=Nb+sna&XjiXvonyTnhsAHO+&Af2UHJABlCSl7<{qO8{pzc*b)~Q-3k5 z^dh1Z$y`&zE*W9IYnWQs?0w_wL-_$_lax!&V6bv~AQU_niZT*2AFJB9vjj0lc-+=1 ztF9zULb_3++5f}dJ9KB(cH!Dd#j4o0lZtKIwr$(CE4Ed!bH{cnw(V4$++Uuz@r~Wv ze_(f3bB(cD*Syv|&*R9MXJKk-7jUW)5aTFj zP({E}N)}T6qme)^Lm!g7wbq#>eti<7@yvrI9!B4Q5U2J5wfG!(%FdjCAQnI;0^6a+ zj~N()&C*X;5>v75s$Gsxo)~W>fJ~PVK_FF-FZOHL7mfRbB^sBn#jwD8c54!fRI?co zv+_%wi6pP`s)X8)M(xB28y;+t?_E_3cNp2gZ)oHu^H+rY*p~`Zo$O zhu)SBw-+iYD4mf2=6q37QSRwbCNVIVacG^4sy5S3`7B(D26hLu*>S#^GZI;+Q!U?I zoF|>Ui4{G|*0L#_AA(q=6}%liN^`elshAjbJPCFtVfaOkvfr(!W)F@iN;-anZygO5 zceg(dTypAZTs!Kb?k{k`|Clot#@4x67^F~km|RXpaxu~>#I6`-6QQd4)1@PH<~fVj z48@U_V%IaJK4&OW@hp#fpDgzadDm&WrO%B!3Z)EFct6N7{w{9s0l_KblMKOtc2L zms*FWA@ER;NO;oAt#VF>P1(b~mt;lHgWDbbfl0L!5s&&C-0Efr=ogH<*NYXrv|ZCa z#BUI91Y^U4?5?-HkNN{JTLa;<^k|#|0MKq6~GS+Ev@bQ=Y0D*c~^}Hr3Ca1!sCJ^A-Do7A)HO8aET)DS?BQgO- zMH)QAl;@53V3&Dz^-aLVqh3>-*_VJ%$nsaf+siiaMBbk-4rTDw&i35jWygwA@Yl$v z9q=mqtG%ME8G|_^(LppQgKj2N*G;Gz&F;mn6f2OtMZpUV{Cnsd;52C;L6(Ys$x*|a z*&4FIqyDr1#9CWbWGoc`+7C+H18EMXO~S@UnvoKW>%?+&_+U{~^h?ioR0L(OvaNot z2)R-8-}5*so!wTVvUtjr)`2mPPWFsJyM^a9Jsch`TANRZ_&QuiO2@IJ|4S3+zB zd3_S;{cQ+YWWZ+CfwRp-nHby7f<|S8bIm1H&f(x_LAeQ1*ljOk#d)z133y&vIA{g- zY-7#8UxI&cp~bKKged4}`;o*i(D0g=9*C)}uLwqt;_ODyySuUPQzR6(_;s5#Ve=Pq z=BkPuU0!N__*NnIMs9#o;-1WU=C3@&+Sw-!e2VWBfTZCTQ=WvHsl0-+TRo2ub~T@f zVyV@K=Q_CW$|=>V_0$`;N#BH`1n=7E5@P*Ty3=HxG6^s{DGhY z1j738WwOhE2#o=*346DJli3=irXoUy?#)DMdGc{_Ma4gHhpXf1R-~JeCJuj7XzEuW zblx#){7SQATk4Ean-ShMXrbnOW8 zs}KYdQJz}i;1j!~Zs5+UfZoyfA{LMsD?P>gSXT~ePkMi`hx{2>IgX=h z%B&TWm5{eLQ-Un7EX02W`~g%0QWB1?KJDLw{aqAzuHpPvzfEH+DN+nL9CW>~)Y zoeYQQlGrrSAb$FvSHCp?z92|DBV}cy?D?N*>Z1kPhzDrd( zPw3jUa{t_`q2PGRJ;UnVMcK!;B^Q5+&JkN()A6HsSyQfi(O??QFocbnB|*{Git3Hr zH+dT?S^PQm5G>-<+y;SIA;nM&@DQUHwAwg)8WT#YxGG-GTKg4^R5k-oeX44U=Qn&% zU^6Nnu1O@)X|Um!Xjt25Y@)7KQF}O?x<<*(fprxPllc;?=8_y`N36jcEogF|&5N{z zK0gra01p+?FW``@!Y$bf#oNGoZ$>159-%`sEqt-H{d+LxTt-+Ul(n) z_-$sciEb@$?uR;eg_SFdl~GXD3pzl7a3nnv+QMS!i=sU9oDw9BpF}+aN+HqfZL}SF zT8;LrU?RZn1XX>vC0Nv@3ca%O+UkCca4 z$F-MEXi=^GizIOrO>OI5m+UUP>!5Sng_SO;vz!^N*^i!=oD4HN`@VD^NCu7y{Uvtw zORd#B`Qz8_eyzGQAI)Z^B^jG?=S6+EbBl$q?fI7;Y250Sdwty6vqTzvTNP|%^a;Xw1UNH?gqha&e+Q#|L zu4(ZTuixavf!3A?wTSi|QqP{D7A_TeJk}&k(5NL3W{GNaW4Wp!&f{2e>89ejgkAUI zwVy|F>>)Pep@)gfwh1~$f7{i)KK%C3OuVLAkwcW5hG>?{&1jz1XAZ6ilznH8t=k?M z!~Gh_v)hR92yOa6ypHow+bf8~)3Hlo7MlpX?3J8FxL?#6IJ+=bRJJR2ZMLa+uosc> zSt6u2@19Vr1ptvnJa4^s7kk;bktRDuU8|s$0HoZ_hPo8+A9^v93_PS0DKfk$65Q3| zJcBp+qKD+3yiQl3r6tgD227*OyRHx+ubsqzGi_h z0W=dKrl#l7l{ploz~Lr|!_t&Fr&(#jgJsQr&^JfzXK3hosS?QC`lsU_|#oFif5B$ ztS(aBWv87|{KkSsB6i=Q%CRu9Ni=bq(;1DIty|XLwz7`ThGg@Z# zGpqPtkBN>BoU#P3C=|p(r?W|&> z($i9L!!7YfSL-40VmfeRk^|y1sz(O#RE0_UvQtvGgF{@an9y<45ixw!-*OX`Gy-cI zpM2lgR%A^@!-WjXGv9yU#O`2aI>m0OHnjhi*NU%uGA@>;Dr0WR8WToidZNimF;vIX zBl66_D1bbTD&~xnOYFm5DGXP|8;n+RQXfUl8jYo$G-aFYKUtr*k1FPtxxir#euk&k z!NQDozGt6Sae5P@IagEujrFUMa@nE@&qt>V8NTpI*7kv3I0Z2K3u#uBwE($72Za~T zD1+>tIXh($Rv5C7=xGlA-=L&Aw_Kbb6N3e5BMX-75nx`QmqY>5L1j&Zmp>OVd>^dq zrH+cAU5flJhLrjw(?~mi916a-?L$WfCfgzyFk+6DOIMN zJ^Eis&S`f3QAj=A29ACK5XY&GH!}s*tBP#X0Y!mP9K zxYjo@e(z*uX9?X0tlm5(*-L<5~0}ct#n-Kix%J!*4eZ%CtLjf3OtSE~OHcbEwuhR|cc*CZ`Ih{8xM zYK$2FdCb2VldOiSLgxe5sW-QI3rs7zYpX<6cnwuOrAgUdl)`izkLfxb_@FNvK$!CB z+D!eryMtAeQo|Trap(M;+s0UYd8kGMywWBoP5q3y4mN?De4^io* zdyZJu45FW}G(#N21fDP5o?P!;Vv8Yk%#2;79E$z#9=<#E&lkI=ow0%m)CR;|(FnYVABm^P+r*yZvl&QF3@fUx>Y4uvRU z1zBf=Wi-IYgm$u2hnL8Yg0EDEf)d|SVEPS>I4Uvj0gQ8LtD?*0$QgFvpd*-6!o4Jt zjXwLXqrh%xdJN2qI9yo-1iEHg8GW`ZtZU%39=ixi-k#)UHOnD7GJ!35Lu2-Gpr#g zhL!r_GL98;;fj+1Qd}j5_cS>uu?8KOVSNKbw@i_Z5hK|@9R{n5>ME+UOw}p?t+PA% zskgx?C$vU@l#)kS0|72yfL(-a-y^=4b~&rcN?)VoBZWI$F-ge^slx(XI-FKrO|*)p zM5eDZ5dpM2SD6)i_3FFS@Jq>9GV&S%v}MTGgf;cfvfC?5aa2`i^SniJa|T`U11Sck z*T7w#UL#EyJnxGoRo)%8W@HUT#C~gZm9Yw%x8s;)a;IL!-(mp~!d|8?AGZ&{IMD6? zkC@hh#M}V~%ez9%zY<0V`)>(=0hoZ<{5C=eM%6!@rFqC4r(Cao;w-F;^NnPd-arIn z=tINlGPbo12sJp$a=J1k-^$eC2v(pND9-ZbyLMWQo-DBX z3HZ>_Q8O_!5)~l7xH} zmQ)auR4%PH`QbO~VInANOq5rlJ^Lo*a(|`pCe@s4@YsY6tUWnC+A+K91XvF3{-eh! zZAf{AWQw7q9;6fm1>vFwokdR_RbJBS916`ajGuO-A$U2J{$|x4BL~J{a#6foqUsvr z2)e?vXB-?PoIVfm3+fB7J4#DzTO`h6{G<@uT)#2s11B1RcS1gZa=VZ z78`9T7WJ*hg}m+Jrr$m~dU_w@@vLa3oZ(9bJ=q%TTvu{6S#l8%;V6k~4gC*|kr47l z!HPUqLrj#Q7y7FbR9QwlbwG?kOhe1#jM3m^^TGTc%Q6nAv7pF?vUJOS@)DU zba!kk3~mP_?ZN_dHVT-!T~Z*^__U-lP594O>uJ>iC0^4r>e|$k0L=*Kh6x#*{H82I z%qARTV|=P1h>#A~lmRpXf;kes{3wW%o04xRlr85xo+QnDrGL%SEr@8Wagl;>REAm3~6G zdw3XYEt%+WMv;WX^)}uRp|Yd^TIiRwu&2s2?DJq(QXGJE)L^m8~whil1vB#P9WEH%&$6aZ#*pRh*v_FXZF3 zF@Sk1bLc<4O?67GdRCQ&TNNW(%ImkzxnfIrExjI^iXNTkvaI&-SLdlV(+jlq)UnMf{Ap#%Bi~8n#0V+JwMB8qKLD$BkTH zs0}=x?-@R=C+Z!HTCMSf`-7*hYB|z6EeXI`N+fh1t?wdwp3K#Ir%D&2H_yJPHtQum zq4xlY>Iua8*G`s6g(2Fp=lwhWT~2P69rI}_?>+312Io4F)kaNAb$dqFHzi8bXbSKY z=yAQee7v>NfS|MOyAWtCHW$)&-c%$1UY5vY82rS_P*#pqk7NgaarZ;7I)kuTwx~)( z25waa`St#U4zFmWGHB;z1`*xgsFr^+;mu>-qgq&@BCwl_=pwb}8i_*!R|W2@I}xK9 z!)_A+N#3Sn>ls-cL{RerCrCL1q90?LC(;&aqjQK2PM%($^YlE75T~Ze)1EXxBVaj@ z=mJ)I?(j&Bjd-B5j@`YznAsxW1AIQ$6OIBqX~l~xq(BO{KoP--&+3?R*7+hOQ8eF; z_2zhzF?%JEE@wh-`cqu@kf}CXLiB-U!Mia^+M}a*^~SU~FXJ+!0cmJ}nqHt_?S#~m zEpGT8_yGZ)#NSZr-F=eT`4>YO35@OO-tJ`CO3$Z@EA7alV_P#_?EVTn&B@9jN`s!p zU*mEjRwr96)8pCpyt=W{!A=P10;~%G3UY3pClO_g25i~w3b^P0ygm@v9-oZ@a>M@@ z!5tr3aInIbt))WbP`=>|5?P+V8ULOdgC}T3ZTpoL%wjaG=b#Rw53Tb4bWgFMe?gU7WRIWll?=rcxNvZs6m2?G1SROJ(qTNI6R2!D(&L);F zks7BqW+z%V=L#6#X0AjJhMYbi<)jis_F5#W_}J(XG4z%XI*b`Aw%1iK6$!BN*~4Bk z=JDPRnvFA9Cx0BgP~an~n}IGw8zzzZJ2B-h@M1%>ifqNq13sVLwk8eydEVMc6~S(< zKeA(0JQQ^s&%ZG%mpcsM^Se9JcMc3+_zenERbfS#@~eaqI%Gqq&%Rw-g%yW`x5fYV z!mgAJ`JkR@CTE4z&r%g@)Z99E=>BHOBi8K@6dcixuvFWLtRVd6lUII5xeGn-U4kfL z1uMbMuMwQ@q zGbdl2E4d?9nf8dq^^X z<5wdK6R@dyu_= z`+eRk_n6Fp)|I%WtOvcnbnCV|PD@UN-!RJ&S{wUs=uQYsYGe5_xga}|7F+9RU!Hf3 zV)mTWfd%~!EtD1E>Ix5ZBzy@~^MG?Q&6R7G z6pNa)8Re+yl-$1lWkg{v`>9@?VyvPnV4J}~FkaT~-J0y%*d1W!aNBj06W`!6*sFO5 zd0m3Ig70iKd0pm2e!g5(O}{t^0)DYA)Kb|~fl*sHQay}Y;R6I>tKY0_X9Z{q~6>4P7cQAbSCB%0~eh%A|J;$4oqNaa(aU>7INibEW@CUFj?mwTvXC#)M z4;*jWC}IBaHi}axEGct<@irg6vpF7j9yXtazUCsOLOwcq&F^VJNk(sJ{~W72>dXj5#G$1(3^^%}DOcf2w0&w`mDCP6E=fl@qJ$AqZ7A%A7y$s^u})IFW8v>yayP;lA8GqhN*>z|FOM)jTKp9a7lB z>8P-8;M0(|JS^(WbR?oV`tiN@Y1w!LdNO-<`UC(0F+2FL0a_G+0fGX5X7k6zA62rL2tL_p}Wbr6uAuGF&gE6ofw^ti$t6u+{ zgDex0aMVS2-W8vl=@1`HCaL=ZdoXAkbluMXJudHL*=zyrFFZmX3?Nl?86fGnfDv7T z=^wzC@tNZ zf|8nzWw$@^{HX`I`?wj8MomsE_f&Ro&lU@80UWzPS`)$aTOeh~sNBbB3gw{B=WOmr z=N;j6*PQfj!kJtW;r-!@Nr6Bw`J>4j)R3QSMhHKK_~~U`W89mjnQbqu>@K8Z%Gt_D zRt++aIl}Jn(d$gTXvrE&5XSw@NrB(@cl0zeLDH@H%Dqq?C1-x^ajj9&vVQ?reKw9;oeqUQ{@tnUo=&Y#?39pEI-x9? z)^*2QByY%$nEALn(55vKqcpacN&XaCpQq^Pe14|CN%?@=0)b90Ah3;NdC<8SmZjS( zoVR3sgV~y=#Ml@6*zw^H^60br!3QBpX2KO!9Qqm$e0%(8mUKC`Mb^T*xQQqyi*>$! z$xxb)0Fu1K3w(KOM*yoB%QF1FlcAIV`L`uO6CB+%?Ea$11V8^2u7ppf9%q90VLBHA zpo2xsNA9tbw)6edKV*;4=37P#w0Od>*rx`)yswMdF)9rKAf*IUeE|Qx#ooFL-2+g7 zko8Jnuz&D9|NYfA$%Otb87`ALsX`9mh<9LNh8eW^tdbSM!8*F`1IVdrw4akq-%%;n zgV7>8?mDDMhz=$vlO9u`&U>e0U}QYwGJ-a25V_e@%hMYZN^9FJh1 zBlIW}ONx8@oUHhrZt{y6+dPjb_j{h8IgTEhT^jHfwFn}&5~g}pRAXKVS+*tTLm48+ zeTU0<6>}Nn< zz>HY6KjNv7`2m>`PO7jVB*RVqAjuAFA#*$o8izF_<2PBBlgrz~)zRDa>udL9lmfKLhOteWnrQ zj0;+i4sP-w^oRAfB%(j1NR*t{K*dtgyw)RB+I;MDenu zU2bq|()S7ma8_cKEVITTJaWqbqO;J$*n8H!t_30rPLa@-*n4ISD0U0h9MmW3eP%yN zR$`t6!$-uB1U{~&x5Bq!fj6{YoLgT$bK)R#gRG`}-$CkCSh>%pM>K1`tH;Puf6{Bv z0T=Myc0Lzx(LSe{Iw6HbbmI<9Q@7@^*qQNEeGS?{!RSP^A4;$dqK2(Z_Xb*mG7K_& zVn0mmc%7oC?nGuJhoq8|L!+;ke=hH^fBA0J|+Ze`3GT;C+`8Q!e*xuw$J z?e$FX&leEo%wQ?WB72nmUI><@rd*7K!-2NMn;rk%AnNexDUXKQ8-;RhYZWwuWMBr@ zpXwEWSAOs{xn~M|CGvdpW%k9sj9{d^JbJhtHVw$X2KZv|V|QaM+#XGqr}(?Ot(9vH zy)P7aG4L^*G)-#l8Ik(w=b<03QEXpCS}^5?=!}FmxFL91M2zMMBHb-p z$NEMXW=boFu_ZSRn(X(;fZ>SHk+HWAA|BlR9$s1U~ZQa zFG`E}?Tk~KkL9Eqcm285*1Y1jrH`~o)-Db$@fve>n%nh-E8-L)OElO?MnZ;ZDArbq z7E~tM`0=w>4whRzY%^89pWn!|E=EYZSf~3q`9jwO5K*8aLO~PlATkZ$`$ceJ;~?noz>zq%Wd`@fVa``pFMs7y9i^?rl$hf(E;{4U1h$a z`&85d*Z}EdGyy0&#v-Yqfuwz{CwFmM#CW^8rU-v-ABjpNA1s}cpCan_R_W#WS^c%D z8D<1YK?09Z@?(nh!Sw~&rkTjrsdDOcpY4Z>2fUx5430>HmTJaQ&C2xP1x{pWiaW@^ zKf8i7!&dAh(!ne8m7sRi=ICOILD-i%>UOstpzJW(-d;IWx7e&~?_LS{ary0*fw!|&6DQcsI0GQk?#+N#=l19OqirH+id5w{Q$Yh>xTk!D z)fgi{md(Z1oDYZZF=rYm{A^(e=xXvPMgtvEqEjxN;)quK?HRAH6}_ndNgbS#^$qyW z&~{w!36xi3$@@w?X4U+zdw{wc*+j{%hP$Cy2nEVCbi z9lT>d$9=k-@mCF|5ikP_(|2QU7K87Sn#6b;wsp;23czC%&lRteIrbHJwS@)Caxz$~ z_}N>(MNb!n0FGNId{UZ&)|0T;&B;k{BZLKB3n3-k7==aVpAd)rtVeW}dcIlzlRf>e zUSCIc3JY!EaLuJxv;2`R>3WGin&B=iv6dn*cA?fnn*8!FzxF}(6Ny|gH!@dWOc*Va z;=w&Tx#{gaWAJ-hOihC0;wP*V*0`q$mT#73M|cH=Old+S?>Gu&1sr}I-X$LE=0a?V9(qLDpGPt?@K*=cQKZ3)w&KV6JchSHm!w$K)SMakQ zfyo&@b#n8Q0&ejewwFQi@Jn=Vm=&R5L{*pjKBwB(K=B@Ngg=b4dOkUZcp7WSQVS45 zYYfS&tP7F`R-(`}mLI0$-CzjFrnPUZE+z`fHeNsh>W(y2wP53kF_P1l(PfBk$JVMP zwV-S)BZyq;9lK7cF}cKk?pq6pgIr)cDrJJ{C>f3;uV`hZHB^Z(LTw-84Cj;_1T=ua z$0Yxz6-Nx`NlxF@o7O^M{u3>|OZK49%QR2sX*0sZQNOSz+h z!HNRWexeq+AP+>OKSDuo5(-WnDlUBx$x)R)Htyy3stxmE;m+~>I_JzvSv}Rr@6EPC zlf8JLq82wKm+?~iESfbxuYl!X%)bPLBJarG=C*eW7q-bCgwVeJJH*#jP1`MLO_v0= zpqzKl=CifEye)-{e4We&GnwT;US4SA8$k7TyM1zgt6ttdus2}(h``-qa;%T}WJDB~ zAMqKIsO#8F4J$BVtQp`sBGOteAZwP)^+ZHPPoRI71+K_&x5ra?xwyOgo}c=_^8Kh+@uHnX zmo_0|M?u=tq%Z^yQqVCF$T2GEBEl=g{thbPnh7|^Cv~%it=jPkOE6)DY{6T zw!B38SUkPpK~hYj8bezjk;CAu`*0|Qwn)c|p(I4yiV29(5HZog<6|xZFv_~{@y$JY zE7@oKt_p2=L0FB9(!w;`QX~Dnum&9Oh`)>c8_8 zL`+lYOkmu`z*|^t=qcVMcMF0hiwRend-eOdMUhG&TeFT(@r68KP@~-oxxg6;=E1ed zPpgyjV7%|~L`Rm6%G!c$mJ?}+qdF@|WR=Xr(La$JHAUqhW>s)G<#F>XsxH12bP52O z%!m|ln=3>Io$(}m*Hf2yy-K0NW0%*RiLtcgm_rXua4N8$hR#-o<});N*1Sv$S^$NTSexX_2JY#MxpYj~h|)PZL6(=ftC z3>)g8@C-b2ANVDNzQiKNp33LHm%xtL9gq+33fg}r8A#z2$)SWPwpb5BGJ3kyE+k+z z=*LNb7^86|s~ERf2lB2o&a{q%@&K~9us!i1qPMoq;` zy<=9bp@WidPqAj;he81WIWj0#&>w+_py~1SK7JH z0dwNIhZsec$>5x}M*x#2lOwcNl-Ia9##2}j>HaZ&m?$;ckY?q0S-q3k&sNAuE7<;nAz!Y~3$mQD!Sn>UCPQNxqJO@1AuPGZ@ z|LlHAgAI(y8+D;Z$T?YfJ{=f*5TC)y)Ru;H)AUR4)Jd0+Vj{lw=)T#u21&ab|7dF} z&Bw{zUleWl{beHIS4|%qY_|BTCbn!B5j_J5ys#j)@gkDqR#Md84#SF0ndJJB6MfyY zKsKNEcjKSfG#-+pDZF{FO-#cq=x_tRyK0Nr_?Ler&UWGB&}s_HN~68|aO|0X3b2r$ z{k&;Fkql_7ha%f00BNl57pvzWB3i9BKC#;)gRbc?ubZF#r-=F9U>PqFti=aZ@zIe^ zK!>!p86Y)Hs*XY|i#dc{Nk*lyziTV%yt5JMdmS)Tf8s6KGJW^~p!E^4G(ce0#)jQP zl2|YMNmhMb328phm#xrUtvhGyY#~BXfh9T zhNoy|!2)T&wJw)2;oR&>y;);hG1CwFW`=0=UNsU=Q0E0Hkd zh9uAeP(_wvNmbnUD?5SMrjo zmQA?_dZ-T#9pJ+q!y$JwL)Mz*|eJ*#hHUL#ffXBuM|=no*t$-BIe3h9bi-gyt@ zXP3WPg1erl3o)q-U7XRobrQHjuz~M)ph$P!je9JA9a!udFfZnJWn2|bTozKlTHS=v zh=cckx+o?9&Ifhhv_oAMjU?TG`5~Oj@~%6r_LOz)I+O3ZqD4A+5sC15L3v1GCWO9^ z)H~i+9G>>qqKmX=U_G~ATTSd|Qxj7KwJb+VS0++bHFk)3yu z^mlbT9EC+&-T>QyLemXGC1DHl4_|!@*QoCP#G%2pbk7v8KEoPl-{yl0PB_=W$H(SX zEo*F2)+Bk|$0skW2H1e4#&KPIEW5b@vS$zOy1$LWMo)I|2hWqsnf*66sn^ow(9OMO z`G&zq9e1qP#*?syeE^3?LuBgeX(p$$$7enii;a1GQ1j0|YTMEjIZt};@>$kOv_&?^ z0lD5^`1Xx%-X+C1WtQH}Oyw^Qb|`o7`Ha@wt&UW9nHI;{rPO3r$9;V{n}$YSJRx%N z8?3UGr1eeM)|tHyXy>M+jb0wTi=!@ErlkUFKH7L&oqVtJ3(O|^>&i?nw>`c^%ZqI* z5*fz@ag7jU!Rk6FovtnC@m!(mVFb2sypVD28Hu$myK9VB>v8Mfe&y0&^_&@KlZBEP zx(?KYYxL|L2w@XxVDxlUb#O5|TLsQHL8+=(q&5Im$It@!^!g=p(~kma|DjT*Z6C9+ zQ;<4r_Yn}6h;}VB{QEkhP-Z(oz~`&x47d^SbXoW{*y9^W&|+sCbTedNctvhWsCVmG zti<~Ry3D_rsXi&}0cdNX4~z)2z2NpVi-g zPP{ybMgihdSmYTysWgkfbN@vB)|y`rR^tq9p8dQ2>j8zpUFy7P`IY1lP8JKh9Qr@b{-~_lRn_#mtpExm z*7h;(=im+@PwL0|2orV17Ohb$PZ|gCsl3ONB11KEs>$LDD)S8Umr`VeY7zQj4~GQA zi@5lm-n!ng$-_Y4L}tw}>GeMH8b9(50olWqIY#L6jYs$O-;nT;>j|z5SNrbq#G1N) z%zF>kTlD_|o^cN)i5Pw0#YJ#rUI&LJXUQ2sI5h2lut2M#Q=2O>lYo7uT8In8MJ5jB zbA`%sZZqNHFj%mDkVlAK^Qta98rrm_~xhJAm;Syk^J8XdYbkLG7Fnj$^BgO)}_}Uivs`q*I#r;e$$Hr`Qlc=XD z*}?;1lsaJRwYp0dsLDV%R$AR9x4wIG(EpZ*5&lX`aT3+VrYUh)Yf-f5_5Y*GB~JMn zZjw9TdBLcLQK|BhDzaQPwFR2g>6j^uL7WS%;?Pl*r8ja>C-0~7tX-?EeIOU!Kwtx6 zZ$pp&Cwq`#Il+eBjcW^&9{C4>_wRZ!Zo}GPx_a0;Z1El>?WR2xL%ZP8mrxi9Y;Bt! z0dX$IFJ$yZj#aTG7aO(T`YJwlO6fg+oqJjfYmb?uiE3%Ss^M3JWBdUdIBQQ>N+N$X zy-#RC2s!<2b`0SG-U_0<{`hX}v=vIJf4qfC02RjaS)Xcapv7&hk+0^gbzDf*9ObWO zwcS}Ka{h2B8VT;(AI{$HUr+5C?+8ECy1yljeCu5NuSWr&4<9Qj9zE+X7}go)_1w61 zkZA%wtne;O*Nl`FJz_*L{6tCK`_i6E7<@|~<|xS|OJ>kh7QH5Ap*oYr2HJg#Y%A=B zVAbhr*_HBHyBRrN4R22K+L-G?bT7KA+t-Ked&=~@{^(}gY|R~Po|99o$3uqBccJ2U z=NOp`_6Wuz`EzqaFN@Xuw88u*OwezVpCbaF@X3|S?|Kcd$@bgmO&$oRuqoc4|3sVC zq46*5eTS269x6yNQ%^Q`32&qKM|pGOtj7FD21mkWpD+H2K3uS!kCkM*9#`Y&VOd75`c$HG$Brc+7b4 zkaE0slI0O*LC2p>oK)RC579T#@%mUtkdZ9&5YK;w*>*nM-&tH|rT-%& zUH>aXras&llxHH5Y~tEYbWjbhpk&u%kY-(HOrho-D5n5}jTPv{O6?1_lHODiccId^ zH1!F>Wy=oXlQ`x}O6W|w++3#mAw_zB`mi@YzV@2nuw#@~UVjySdwMf}0>J-tF=9E) z__Z%&BU|kDZ{v&+kcUUmm8_V|-nU6fAUZD#P1Z+Yk1Ks2=Gbq>TgWbpjS!WQ zXf*pVek)>b#}PIK7Z@-@i} zV!Cn9*8GAE^|8j85qiV<$qbDP4GWcSRoJZD%q23853@v*YU)Oul@ilBR?pEK`~O5?pV}Cp zc3=cz5ECOG(Sv-|g?4%!>w~mjNuFD_bB?z3*+zQfJ;!Rg_>M;?tZf`2mA!HA4iKSK zZDq1Rk@NF`s=EG&*?_(i`7Z<0IGLgVjSApvvlh@LzyWv`0(|D@Z+ie{d0?3ngW37d zS1-Op1MT>X4UR=>0N+?7D4@3fT|jlgO5Dxx4Z;0-uqP^VJqGIs=ns8*MLTY7$8qd#s7LP;~XO;P0n9yhp~LfpozSLK55^y)Pa8|B1pr zyqdmtnkW(PcRqc4qdv+9fZE7@*(Phak$-gtnxrHO2+F(`==`oKIh>scF92L5CWT;yKZhdz>TbKb-LD21Xm(;o1_Z^mit;dIldV5Ekq}i zE7+Rm-r?)wVY`M^Zgh!U#_DFb zK=Wko%`V7pVKxSTMdsieS0EjK|ItQucRNy_>9}v~lI!*O9~Q}v2cqBXe&^Qf;dAt% zr`+fJYy5n#pT3Io!s|ZaOVY|_Xnv|UpFmm7C`aC2alFP#Q+534(n^lM^DyMW$0p@@ zRZ#0n@%`%dp=^;u@{KDQK7Ly8fNBmxeT{*QiqRy*QG3w)Td6nrim*rmr8uV&f_T zBb7K;d5Q$l%PVF4_@{u+Sj~dvTQ&>etC+S>ux5zSP zX0~^*_b@HcmIMssSqS!O9O}Y6+#{jgMR|4LTy>ZTcvpZ#e9gZ;Y9eQ>oD9Akq9f*U z%F<86cEap{(%Z-JXVcIntlPLq0v(IgDd6J|=FVz)!+8NC;~OU2Vi>PC$avLd-NOx$ zB=1s}JJ@}x?bd6bXU;;(tW}inR4FNk_=}%0Uf(trVAH2&L&BhB!kbn+{gfe_92aL5i|Je=(B)mji{{W7M08jM!0Z>4L(RzRx zjbZM`W`GEn(}h3KfAS#V%))30ChoiDE|;0=>jv}Zml+;K_JU0f!VXS29v)yxU3J=0 zE$aMW(M0P`bEPVkj(u`5utCw$qHTr~^W$CT26%q5li%oxp+{jgvz%kH_8Rb1k_mKq zxm^1id<7)TVsMp-^*h(tMr_20hdE&nE2gm(_oMfiZhDv=>BozGi^5beMn!WRnlD7h zj~K?aWMfXVgc6(Y_~e8wV#qF0WX`Wm6GfxKuy8-Wm*2~2@~EfOf+W}>1HMxpA1mc2 z%UN zk`_*xqf@3|Z;oaxhbe49+4JmqkAJ}U9~B9+6-iU-QEWyN=zMM#y!Q|Ns4#ZsV9NiM)T9WS$YhO)AC;4w0XWh>*@Y*A2UZ@0n$5Q$D#$$DJb=%YS7-U6{83r`YCX675p5fuxm(m<=$<5U-8b^*KEd+9uM>$WDGOw1c%gn*VHEY`n$axmCC&Fi_9_rfz&(E?79xX6v@E=;1rwi8 z1`{U_NwNbZ#AjTQQxpwxpzHXvm*9B-9aZgjdOn}#K)tbQh##%1qT_>m8aO2_su~#_ zY^(lQi06WH9hl{1$g_dXyu5&=C0JW5Wj^)ykkMo^#|W&6zCa?H)9Dj5)kb67sx1s^ zjr*a*-;pRusUbJJuzElr63WYDn#tCf$d8u^kr{_+_{C3Vkq-3u}lP8^u7sXLeSE<>kWB5G_c+;667d%dX}-qU`dHjo1U`4B?v=E z!gGF&-Fc05>V-kGRK~*SIJ_|`Cryb;eSBs@C4y{}a~v*R??hvJ<}nn^9o12EOf3wF z_8C`~CtNb7%`JL?_lW<(mO=_4qYFX%T%e@V1#*{{C(pQ|1D(Q#EJ-TnY|Pbxo@l_m zf)LedjM04u01gvfdOSf0y?Xy;_4r(;^U4zF{K%T!Y)D`1^?0plqi29 zjpS29$sZp&R{Z1i-)(l>71&SH_DVPhCpIh z?c}o)IJBWv6cJnmTvyqoAVh~CSTd7YV~oKex#hkuw#2KbRcC7BBg zVjZ8NNf9!AV_|WjboaoLMruo94x?^ogwoCE&C-YMYu>BhP5_}u#AR^Bky<6If_~3o zU@UeZ8={b<*AQ@5fKpqx1&O~*gg5v)A1uUnO9eC`;YxY)c!F=r5P=U@SF4jpBR|=s zk_Y}ZAr<^47OA|WHx$A5=S*!hB4U_CjG$$rhM1GH;4YW#N$imqh~__;`xJsi-o9CX z2ByJkT%hP6ezIn)3Q*d`bl5XhK%G7%M>CoRIs~teF0`S5{mLLbjz?pSw9+cSz*Jz&KY!PbLML3bu?o`2flukraihc}E5g&}N+v%ral8H!&mpmSyw_s_D z6&^MI^L+kfqPpHopdW(t+E0Q7uNWFG=0p{G6LeY#OONW+aF}Aaz7DD!d2#n64&Se4 z%JeZlRxITE2Z%u(a!nKX;9AZULfAh{^4kqZ_bk1NTP@FMP^TS-76*rvPw5Z&%f#|^ zTOdHg64Ipe$e-Oozw@5|{=<;y6?&5;zGzB()nk~E;RfRi(l*?{Im6JDSjtm(o=}n}$*VubM(HcHzqL>jvir&WQCV@?j zB=%{W4#lsJ4o?~jwray_;YWn=_Y`PdBXlH)YBk4H$A%@=G}|xxeU2meYXfUT{fQUw z>h}`+RTn(HY5Im4cn>^Y37~#D69QX$e1pKEzS33hJShm&+(+_=K4(`oJF-Tj;Wh~r zNR*If7+NNd*tJ#3=!fh&g0F^6d~mcd?sf7Y`XhqK!O&POB%POBhK<@p8d9*N5J)Ud z6OpjHRPs~k_VQf(nma`>lBO9hTCoP~>NVt!gC+%z-&a}ys#}c{y*L4lvFj(Lh!Sqc z)^bGR@K`e*)M%kWXaa|si4W7Qu*?M%P=o~h_ZfTK&P8-sQGH!d#XD{~D9F~;2<|Wh zbB-P|bfQ_RdqI6dX%ckY67LMvj*P{d1iY3(iahVCHHzrdO- z&20vX-E|ZhoDyAnP#;TH_**LRzxc)f2*+7_J+Uq3j`UYB1M@@H4s5wO(``r ztH4S>zD0a^^RT>y>b(P||GicKZ5eEE=PQq!t^m`FI%GGVbv!(gkHvQjb>L%K`zU^< zT%dyBwn(6L!ms7L!H{A{wZJFfi=Si=T%)$e?D4sdg&|Xm=0p34od?y<_xy}hW^hPE z1JwK*V<+|5Jqbz%+M*v(c19s61^el;ayzFhTcqjryOuB+6Rgff*BylvoWnPbsi7MZ2b&OPx+$elQ?TtbM`kG~IzY$XMtI#D%`E2N!LfX{S z*QZf9=Nv=Q?8wBJ#P>>)Ap>1(C|49A;2BXEz)c7qbMF%Sax7K5<$HemvP z0mTBBj9}WhK1n0&*w}?%2CZV+{-JJ>%r;UCW=2poa=&I^!{O6Q zlm8@&E8!1P?n4^6T4{8hkyP22t*f&EvtV9a6`kptvGg2JSTMy8^rG}-Y=jc!64-$_ zsI{|KY}QVLGP>k$x{?#f0@cwSFVDwrUrQ^OhYIIaBW*mmBU=XMEZcj zFC+hGZSPU!%NOc~Uy(@)wyq~I)8LoPv$Mz1zaZ$PplYuNkRiaBoTXV9pT|uM==-;8 zf(hQN;qawow2JL=T@g`XJy~ z?Sl31u}))~FF5>XaR&ZnuE`(296qZf06sSXcPF4N?OPFc!rB$CAeiPXXP2^qk-~Z| zm=_)rMq-3|7JuthljKF+#{doU{nO(|00-etQa-?#^0Uq6GV>NGpwn}>Hj#8_CXgU- z$FT@3`QkGAo`ADiUNRm>ishs(UUQ2_QITi&RHgID0(7P;$rZV6HO+>Rh*x$2+Z_E2VWq z$9~V&&y5)X7f$a{fD2zP9UfVz7QCAG9qvh*-)_L$ao^0JP6?pslIWi-7*wqWBAmfl z<`L5OtBq8~@c6}VV?HD0C1DUzl6r`OE1Y!E$E4KA3N+u7Q5v~XOOTa@__}3-Xj0*J zv+1&|yBP5aj509)_ZYD16KVL76z=A5lgH&~Git9QWs*wgrC`G2M9l9A8nj}AE?F8K zI6gnC-&T0D>CDb?EC{EU@thI;-+Uhs({FWMkkW5_G#Pmvo5LxVv(h92;X=h-T`1j{ zT05D~6s9LKl!sz2QLA=6PyQgh3yg$Q2QJ<9dtd!FA@)(Q_(~*nq^`p1$|%DnWr>kX zo{jW!jdWDe>?*8HWN?9?V$EpW#)DJ#iOgjRT#PFv%w{&f?%Du z#WhPrR2@i3DfHS`1vJ=&R$y(w7gl1ZrpSzz8`%DN zNlGKe3r$lDmX1C1Z8cr_VD*w`AC@LJ`m?7n96(ms8gsF!4h6)c1gJBGq_Ujn* z_e=xBVAY0@dzV;Kr_pc6xcyW2>+rD!X^DZy5D}fJLdZ#6nR|`@20J}$$d1PLE%`=+ zEXM?IoFyzHv|UQ3wNmb)lfQig*`CAPc7KmOB4vMNGt^BX>)zW)?5m|>sL-RPsV+o@ z?)I(+Ad7ncATl8aXC@xITEZh!1NPWELSa;G=q$tvEo}`3^I(UQ9k74Jwl0iwBfx;S zB;Pqsj?yA9jk0Do8eS#@sHm#Ez%F~IsDwa~{5xrsB-dRKNNR#B)P%|{rbxW5iT%bX zafg!`Mo6a&b)E_Oepz=?Q>fygzJ~WsY)-F=xJ-gBqgpCti`R~BN1!zL^o?ccXD z-b|<@wxhJ4xJF#7r0O(TKGHf14ZrdF(uRHEw;DV&RCr1jc`ne-Fa%FJoxaS{!?>d1>>+kny}1QF-q`d3btC$R znKn-FDsg&p|1S|)OoTRV`1m^H6C!b9N$RJlD=^E=BI!UuSO?Lf+-TBYqOT@%_2zt3 z$rBma4%sB7LXC=gTKl|WhpGYT6Llm;jRi%Xjt_J~SFjLwz67X`OcZ9&)Llu->- z4uRlLp(m(vrJEHc1_6)@!UtDJOg{Rj4;JY5#ex@jN9i|Fgp58_r=_N;Y#Csj>}A7L zgK8f=7nvjxM~R0o#X7K4rd3>=9?-7cp$7N6&Ay=>8bt<8@n-CQNVeBq%w~3$0_Y4$ zVCjFr5Xa++4E8wTBCpG2F2(nJk6r$<^x^R&P8tw-lnU23YMDqRUnMWo-Iq=}qOaZK zq@S35<`yH{Z&iyw1ew_rbgu-*Q$h|tI;eq-u-2GqdP3mU zpluiY{c=CUEMc}){bXp4v-D*XC(WA|iXaFAbdyC$jD}Y7XLzFEMfB`rYtAuv<8 z$0h^TCx2tYk@W3=@i>BRwH&P&gRB!#8V5Kyf`65?Dqu|Yy^hXgD14j#|_51dSiF*5C_IE)fjXoz%L6a#Rj>gM6hMMt|3p`!eiqf$US=bJe%G584R zhfTzKjUsja>$c=wNM;l{%2b^S7SZCC9=t9*uAr)+Zhc0GkHKO>tH(r<p3{iyDau+!?HU>IXW^sE^mB#c z+Y8{&X+!btCZ~L<5H-ja+C8mE-dRLw6|dBmOiJrZ?9-MOLf-9c7QnT6l&^qP+K!WJ zXtB|d>0>1JRaFZUa-zgQ9ZRN1c|R1oJ|@@djU z60yl$GK-V&v4ft!1#xR-deIlQLxnc>0ps9`FR6fp3z}~IBpA3zB!`{Sv$DwA6ct0^ z&B7f?><$-e7k`DNZPrK3u9&b>T9n(iw~tu@-Ule#!rMRvrpbRtU>?p@v#5{0^eTqn zp|jWjX_w_QrneA4U|}01xooakDrJ=9JXx12RLhjM;T^NpJ|*%Vxce-_u_K=c8FpfN zivFFpOWZkD7?|O&b}-Q!BSKN+#NWp5w2Pi2bGUv+<*^HG&KQ#Fr^G%GI-$7)E#zi- zJj|%ML0JUzQb#|WLH@`1H}RajXn=tQsT2C`L*=P%)6pNsGdJojLt$z|T}a~JqxLvi zO-d?YTuNls&h#MY0WbZDP^42hrZ7C9SFqy zx2~Ek0gKZ(A4PX(Kd$^ne0=euyWw=K3@S7e=5z}sDp4_6DXl9>TnsJk@$XSw71K5H zt2`boUY>tkSuCpIY?1$>1`Sq7lm%>RWn3VYc`oTQ^7rT=$*TNWNP=?vR^9Ui<71t@ zUWu_;sL?F?v$ZCaEp#{OQ3U)FJ!cz@T}w5r>znn`L|;u{m8#9Q)nkrXa8|x?OFX)XLd70v4L+4ST zSfLa${u|;l%-kNH*YNILctSajA&7b%p_L#c!gYzAqsZJW^I*Q<#FL1$=1SDK$t4d=Zzk(d)5m zd7#!n1t(~%=-^@5Om-8qP7GeD|8Fxc6`Vi^#?U5nG$akLgVvOGz0s-F&l-g4MfFXB;nnSmlxB;z&2QjS4~NBQ2@%5rDjyh?>%VaFVNjq z>11fh=lU2-BfAA6@mVpOk6Wjj3_B)KowyNY#{wU^r>6YYhg!5XHc(O8Qh@wkF_ z2aZ!@e1r2IpxT{j&g-cz_>>RpGGz?RasAIY7k~0SWVji+NwQzJq%z9DHWMy0?W)6{BR?@ zmC?XHjjqns1A`4?Va!Stm3CPxfr&t0Ki#EoE8msewppc0cFTtWc^56OIm$4AE}A#Ds4kkX(HMV-oolW>W0b1A^QV{0m4{^2`vF3V$X zeo>R{Byl{P1wVGC$$-LJyZjN={!E?yhhsE>U*`%lI+Qi_HY_47S4tPew)sXYd`?Bq zlbbmz5ee(0l$+(%LijP4$Ln^I$hiL|c5T+4t8kA&f}2l5$$dPKNN=gz=&Ko5x>)l~ z$incW(p899bH*fgS9Q0%MfDKI^?Xu@-J#l91{+<;hb-y0>Bv^?ls1IGw_Dx_~J-#^4i{VO|pp-Cy4S0d+|g9xp=I2wREr zRNf^NVhB9#*r0bI2vY_^FW5L_ye*=Ka6Vg-#^s2DaQ)7fCTqoy?UhP9i(Rxy?6XkV zvLF8=`O*`Aw92M$^`FLG*sEw5|E#R&Y9*5m3A-COX%91<>5sDn>0t9=J0=~;pZ{m* zmlG36yN5t=KZT)e*0+fgQ#%X9yD4*RCqh*_**mENrVcS~yIZaFPR~XXtK|pF9)Y|_ zHDHfX?8gX2&wH(34#4xaMh$02@G@rq%hA1|{7Zw_03*i1r=#Pt7DW9CIC42fO|Y8H zoYIsXIZ=5EMNQ62YzDF|U7T-zzlCmVGboOFQ`Ne(%REHS;bhcOT-0*=>{r9Sv0(-I~MsH7+g)=0IMQD+A$M4Z>)Q1ani-3JY%d#*TvWwA4k z&r`E_>89;JkyB~TcQxrB^HU7U~RA(Wqg{`BN(Lb~&*$?v~?V7}NQIVnr3Xc58 z%^cFnA>WkK-9eqE-dK?=eJ+`o`2y*}fKE~Y6giyvQv;1W4*=~~Si3)+`yB;^7@}>U z#L1=oJWcWkbg^10d^_e=-7lAqW+yo*=4`mMo98kl^{RinY{Zf?R{Rc+hJSzuIhF1gf0n5f3l|pXrza%bBrHxbx zag^tv9O>u^>Zn?9777d-nLH=*ZC*#l3m9LUXu;N@xFcbcV_wk0yi|pNRj{TowOitw z_O-mdRZaGR_C&9#%GU3@<7DQxc2G(_I8QsU&!J);pPub5If_E#A zAwYGYE=M;;kJqRb+`f!(r{?VS7ijk;{-#s&KC+D$G1#2<$5nMM=lSHx*eHnqM!pZPYQU5vQ5VRws_BJT+4+CYkT@Tb@HJF}?kZEl4t#ml?Li@Ru|SQPGStus9F z@3M+yFj{)K z^1l%m*bq#c;O6#- zAw$l0H*!2pG8&DLJ)|NXQBhdeQ&!MCVKq%+UAa6{S@vFSFSdFUEkDKQeY$Sr0nE6U zT={q00Qx@jrT|X_?|e_UFFenH@4g>&N}%V5$DcUk5n4}gPg?kX(tL0vvgW8LPuAnW zE|`>m2-N}OTC%Q@&GRzjArRte{@VqUR3^y9pNJf3R|R&#G;*M#Ljr$vxD2=L?(BF1 z)VKkk$wLh%%o0w`pSTm#fCh@G_uPR!`)m8pKR~l{tcAiaT$+u&uZ0Q*+M-O#CR672 zA`Dnhcp7bKMhQH!&&fkfNXIyr-xVC&q#V%~h}%sONo~C4l4jS4VLApA+kaq@Di{(C zivpf<;#`UsAUt8?#fS443GVO!52r5{2}hG_#9{L6A@3j8$-xTKctF17^;!>N=Vk;j z{dTSQFmzwf@LBkIxFXSmZ7vPocSA_H;c;=7HR*8?cbamy%3!*`V}Y;pJr2;)Ao3}D zM@h%!@%9@2!#A}QU?{NntT1X$kac|8gA2&~Vln!<^KM^zt#R|7i;LI!`2rkHwqzLH zyzFcc@UB^91Nyd;{?7m~B8tE)*#AEP7#R!v`rwerQO~G7(D~pxQ0)9hIQi95+7KgU zP^L|1A^rYg@Ph7@FKac*4x{eliE^?CkqsoG^6G8i;@%evtN z%j5`XMJ43%=i$f|1xn%48pLVob-1*c^@&%}x&|Ur!+q;M`H3QWv>G*>OcolL-)gRZ z(^C9MnDN$F{;A!V)T4cmdpt>~?SM$OGnT=Pgi-|^VJp>??c^ahC zp*jhQBzo%upHw>4_@sp7KK3MrD#Iif((8f`ty6Uy5G9232L1oybALF$WXtD^ra06| z;q#?RDz=$h#1RM~De}LOnt4}DX(>2qk=G!}7j_gDOKq?s!Wwqk`ZX%$y7vR{2k@?U{N?uf z=B-h`a3t<0!RSWjEv3~A?2CI%Sl>Of`zUa$o_xm|?$3NvBYq@^RBh-(`vZl{Sg;oI z`w$)trr;zrJz^LEoo4MH*8H4!4^7L9sRia0$)*^|6jk`$@DHdVERTVZS@I=$4C64J z24Q_=X~seci34}p!1Os`Xn4;A9sx0h0FzFm64Yl7L&gYE(qxAm-8u;5J4P0zu)wsm zKGcLCR8D0Da7Mc+Fr6?Zh}Jmt=Xfyo#A7#xEp@><{gJ(YUci*FfsL#E!$h=T3={j} zODK>v`Z2^KLK#kFl3eB~o58xll|qi*EmUUx&Gkr=eoBWOQw!qJZ|VcYn)Hf|-~SM+ z9)gA&N&G;+?S}`}zzPKsKaz)&&hCLVFtkDFVEjuyWx_qY+L)P4krPd`S;1L}qzJs= z+vQd`j6%zJU%4@8BXUf!vKa24KZroGU>v_{2o$z9v#nccUAArGUb-%a98{L}!gLTl zsuVzHo7{A6?>6R1h~LE@usKkgL1hkURU_x}SM94KBe{wLzdsWvlNyESXWWWfNwzxT zfgvXAEtwiNIe_$1_I8tfDsICb!*Qvi#4{YS9SGp}1v%Y@;QACa(3&1b-G#XC3mpQw zUiD^o1*LE?4&efP7QHN?BcZ(vBdm_qjsRSWeqE{c%B6--A~m2{Kb zA?4chPomQVp(B|w>Pa7-nA29j75<6I@GT?70-f7VI%Z(7m#EG>hNb)fS2sKyQp1~N zIP7;}Ag%CAPzggXcx~FOUy`?pTWa@r7~I1!a{+%9|Hq{f4eL4wlpK(8(;_22S)ji( z*jM7Kn=k00SI~sw)-(faU?tsxd$elOGTPE9SAt)IJyFkGq!8^fH5$srBUbZh5jylOrG*PEcsXcNCGaj^vAb&4L7)IC%(U-%!XhN@zIhD3`q0 zKQ%BOMJxgL;Xsqb%I!fdo2o!G*}9FnFB=r|1xbj-pbGV1r% zHU3}gX!EMQ!Ty*ILii%s!Lj19x9EcJ6I1h|{z4Mv=Fmt^4>A*Dz_a^H<)8)}=iTHo zJQW)br1*qj+Bb8k6~XW2&a;ufqJv0xh{jVgj!R`bOalc$Wrm#XN@&f7B?h}j#~pBQ zEEZDtVKSr174pgkF$&OR;iP+_qr|nzQ#8a46QZ2bit>>i4*!H&fIyh`@$XamZ;2*7 z0)OZ2*yqI2cVl|tI&?e~tt1C!IJ%1>6d$1zMkyAWsP2(&-xRsiT{eG>b6kd?T0P)f zXqjUzFR8puk6e8I{*}m36g_nr8Ggb@tF4iX>9lHHzY;kn>EE%qHC-a~m1+u=_N>+P zYz3hPLz*ZQcN)g6E`-CK;Kuhs2+4>F0R<^VN=-a6%KpYgOc-68buQAK?pTN!C90xU z!;W6c-_*M`dYpa@iftf=BdEr^ULi&b*czY<8xe}ELv$TFBa454&7_APy}n%VHw8H3av$Hl^3LbAgX2mvQ4fQTsWp|?wY7B8ctt2T_gh#M2>LUMz6qD>whjbJj#tK1FTbrPk{`HSM+_c`1!HNVATA)j&7x06L*;XbfA*d-z zd$C?7N`QpOxkO_ySeam^c!*X%d%>H&;aC7!DnE?kP`sp5X&}&*GbgLRvKB$Pw^d)+ zoczA?E0waP2YY#2j*+rf>Y!pVh|-KHHH4c~$QQ_TeU>i{<#*Rr2d_n?@hWSVIbqqmq-gO!-iGF zs3i5lx2F}3N0t9RaWslgkk|ZUco?V&o|v3jMARy91AD1Ez%I6VXySNW+(?_S?nHSh zv7REnZ;n3pD=JgmEM9%+(^@F2P21l2#;#5Kh6L9JqQ&%Gh{nWz1_ZAk#= z^CIQZUO5?;Wex^BwZK7~fk2Y4D$8x9?T_jU4>r4$NP>#RIy5mcJ`qo)5ZFG*;_08J zyyN5#hS!Gj;J;nHB0$z#>VGDeT9H%+#Q$cOo{m9MTOI#tt{9Qf1!06 zF)mF4LM_!9#)V*tDhoz{O&mq{5Y?ORx;;7xuk+SGjV>oGw=Vs{V+yclVj0_Gn zO$q4z{Z>P#OWE`}=}jj?)o|p&Jy$DH7j+7GRN4t`cBs+Op)lIHoLXkMN=~fbreQg| z+_UuH2tL!$QoHrW*RXuF0)&V{Bdp*-wc6-M#21&6&viYIttJ|{`JkjzLaA?w;<(2* z8p$!cgm3{e;Z36)8-eFff{m|^Lq~e@xSycE_dxp7u3;W!P{Tl{gjNNxQE89keD*m@ zqg9Y|K;W_kR*WTvR2V z&pm29ejdLxn0}7*ErdPoJlzE-lQ3`m6cYbOnnZ0$EBU6RUrVwt$c$u#0^QQ48o|@- z=PnZZx!uio4Rz?ne4h(H#Me@-qr^)buQv0nl34VtaV>PM6|3x@*!45v6MTgIWNpCO z{xNo5_|o7ic3CN0v>!RsOwVGbvG~$|}vyAo(8m1}2Ttr!z##Q$`dw-h@R?qRjK_ zJwEL}-ZD!&?EgUt)1u(fSB7~6Kt&s` zrWnR*2z+&^ltH@#*Dvq?3Wu}irc+A5oYt5Znt{Ea;HtH9ri4j`>wH-}Mle zbC1})=i#>)!+~M}a?qCUHGC^&kF*W+U`qE^-k9tT7of~@p285TGGG zS;=37i)Cxn;94O`XjV~_G7Z^F%&4b6?_cCGK$}m&pD{nGs@P?1PxN3ok)4m-TzdLo zohDF8(#(X}v$3UmnY->$nCq@(Q7M6u3nXQ(Wj=*n(C+5|%a05b>%%4$m{xsK%CW!}3r4}0=>g0*0zg*JSbAImksKbz->;Wo7BtFrxQ-w|U;yKZ2KuvtLgE)t+3dD)CDdQ(jiuY~wEL zwco59rkmqWKs(AD4&V6o#MtD%Nn)rU>Q`bVaOrn=5pyub~J^THK38|g#VGnYY zqo|+9)I~9noKth{Q1L`w1jE629cUcOC`)w)lq83n%kgbIYs3_dyl^BZ&AEI`Iy3Z_ z(4=`>ae5y+=^^EQNoi-4F6)$IWIim;ZT|<;i?jmTwMztzA75Pml0PH!d3z;EVa%FZ zuxZ9B-h|zyMTeVGyx)i~{T3Zr%m#){Z0B_B((9JVj3xM*8^WKUB~f*wW6SWit?dk@ z7B@MxW5b53bgj@j{UHk5$_y!2TA%6#wpz3qcFI= z>LuU;q|XhZp(xN1?}GRUQ@inhCVR2Votc3(E3Eb^jiBw5z|oQ6LD;N+(6?&QqHkj{ zU%t0;m8@dZ5ssM(wPl)F-b{6sZDK27=}%`pC6>AxEc6;WatrvCARp68&1tC=RB@Af zm-qeq;wwiTr0b(y?w_7}{3=y2y?3SMdAq-cm(i>C&mK`u*eAmC`JZP!y#MH3yvhJR zdjA4o{vCkf_!j{4p9ElZ9(q*%I{;?kcwjH+6KJlrD+J8Z9tbvHoh-?U(n9r0&AZ3g zX?7mQ0l`5JRCf0W&Cbs(41MUiFyP{JzEl4typ>s5e5c+jf7<&)zs-?0B3Xj2{%bA9 zLs_pts4m2UTZ_&a@7MBcsxVe|NpIVjQHZVD#YA#qoNnOBc1$?s6{;a+^E^ z*6i0Tcq5)=kj-Vy@A@2JI#mQKn!ZF+bymI0x>3%(no%0MXxc2%oP7>h7o2KzQxnm%IBgeIIhdy+6L6S*y-Yz;2&GCVUn|a#=j4$xLm} z6CP#FwI1kRV~oQb6-dm<+NdH*9%HBcccXsY2>td?(kU1HZ}1- z1Va`&?QOu>e_3iYWtN^@R1#oRyX4~y2F`?BUdq~{&@c5Ls3{!j2xVynwM4C?TY7df5@~V*Q zKvo;n00kKSTn%5vET?^KOSc&nK-;m5>!c*c3=*CKO_|wel!1LLXS@MB%6zgQUVL3Q zsq%x+`ik`xDu_IXy=Y;ep8xJs0~d9B+hfR0M2sQY5nPy3F2u_+e^?8*rNXy+ZrS^8 z<8o$}hi1-_Q@wR1OVqfUSWPM#dfxgg=OIoC;aLPiY7}zJPiti(Xrx$OGw*A>l$h~0Y@tD#Tet`W>F+@>;$pgZ*j8I(|+D>S10cueKTTu z8wh)IJB2fTKVFafO-C6F27{Kz3o;PJ{be@U zlMmtc0w>%BZPJ;skr{{va+u;m-F{kNM``m$wJkR-FZ3&{Xb&4h9zfoF>TIxP1Ft3% zdWXF4;v_HID7~}nJQ$hYk+95-U)Orop290=H|jw*$}~X8n)=s>>jzkaq`#?*v5o#` z1u*9ouFxLfmmsa5(_h^hq#8b$%Q~{$4zijVx7T$BU-}fQT7GDH(ha>+^HdS5er#$C z#g^Q3DljS7Z>i2*R;Zt|O{8s1`JsYq9Oh$#YvQ#=mR&(-O`e;8SC7@JhFx9jC#^1I z3|H~9mlYr!YHA7_DcN$Ahzyw<-KPp|+{dMe7~ETFKr|w_qoPsb+|hh}stRp(`EB{F z_E2Qsr9ohzuE{#oh3qL)&B3KNMQV0)8@}X~{i+8Iv0yWk`Qux8C0u1Y6ag;)G*|1* zFM+NG@uP$F^J#3MR4(hLM=s}z@7eFqfzuB_tw)#vI`ixiM%VYQxa*x|l1CHX!Pg%z ze16xpztI3Ku%27by6;y+wRGVeNH>-O0|bP-OJGYSg^+31#{^XFw;M)Azh@=xCN>6T z&+>T6vuNY;H^tQrOsH3ZKS%7{U}&si1_4C%H(Q52OVnWWNq`(ayJdja>%G8dNA3(q z0zmm$kEX!RiPj4|!Bfhe{S6SgT zc1Lh0PtoN3RrkP-X45H#K)dM426lp#D%JfAcB|^s@XpYAw7CXylPS9>EYatMmJq=@=kb`Dvn^ABM!OeMY#?*v>r8%JLp@~K?P-oD)3 z-b9nep>y5qoni`)E2vOGs%yKek`X6OPFwFb4XL59mD=>F;1E`mkA5Ard0z`>#^af1 zlN75BKTj0xW)|SK<8mlG%p6Wg_@=zyEVg{PR|#WMfdN;1P7aqNzoIO^chEljWxJ7MsG1-xur`(ODVIaORe6fF%Chiy+SG|`w0Ic)HJVn!w*iZ3 z|J#jprBb{+q`O)Ii7NC`9Et8P&EiqC{W~>u7HN=m$IyzzUxZ`j!*Vl(X(=IbVKW?n zDjEQ&SrO!&v4TL!W3pLV&T4GGQ4Ay z=LC7>y0UHm)qYDi4xV3h6YTN^X>qPeMn0l?r68p=D z-S5l(nAEx8QR@)L`Jco*ba_{;(!O0HpIYEGnkWtt*azjfHqAZ8k%xCWx8XjKoF&$& zM$3e+fVH=c2X*~o+E9MB!;<8>}M9YpQ@_(j4;J!dkU54jQt0Tq@%zuPM4V~aJQ+#`NYLJpm}>ibvs=-!J#2^P-Budl zlyLrIH}&4{7hl;NpmgKgW3JbW56r6{UWe}rTCe7*$1@6A(ZR;Y8(S1#Np1-K2&;s| z@ntgA^hz>F6c-BPYO=1r&9f#45Um+CS3t;LBCCj#i&GggQmRx;Snb?JGO8<@#>=&* ze|cKb1w2gqahjacW*B$sb6^@R0U?v>BLMCNA0Br$;O*IEjEPQ4Ro~=?wxKe0^pOOt ziiQ*R&jlD1gFNadSlW7u@br3CvRm&$U5YVsts;FYx+FL#+4*Uw%prj^W9vXN7Y;}9 zfaWDM%FWDI%MurCioR1ryn5PPIlK%fSK z4_d4T!{@oym+|NQZ@>H8ed6DKOE9c`a{p_p6)@*_6r{uxBJU0WA+dS zw$aPW&f3oQ*3AsyvBB)^+Y_Mcy}fCFTfjcgD)8(r1{P?RA?R5cir=y05AXlAlS}&B znHTIvW(9FMhQ02wNxzuzZnIVj;h{{%!o-;@U*iJ3Jk`c>D9uB(OpO?~_K6WXHl70Q zP{+kkDnKQxn-=q2uB1yFxwE^(`@>Q#E3bwIlTHJVJ>5o_1qFfGL<01yXB_01vV=XC zM5C0D&b*`MfROmt(GIA(8V77jo+qb;&eAA55aUTg&kfg@x=}YWNW@vt*A5|7%I;3ogyjILY|QRG!z^~eoWF2 zBNNozjBb>+sglNUjbX#CG4`|pa4nQqDD$FOdI19hr>`TUV0B%0w79`R_8m|mMP24` z?7yloRSpQEeolsb-9`g}jry%JR)9>e?pR`!Ilatp<3!|}o~5XaC?`#b0Yj$jRf?EW zFD%1k25HWyOtyu^GZL99?ce++O%cVb#iZk5_}u9XR>SD18XrSyoKJl&JAF2$H_aZj zTixLBTR_>lRR2tRB=t~1=zjQiwW9iNn#;Tb!n^H+p*T{iXO%QE4LT@qgREyY@_BOz zFyz|W@N`VbNoS!jz9OOO)g`ixcb9IS5QG+-|JxSjSAHlK&Tw4aM!p=ESEz`WYKMHDVM|l7prO2 zH?&;Tk_g!*A)k!cBzEe9U5u}bj<}AnBxsnlPD}8C>rG9(scQj3rUnxsQd)v!%>B6>g9%}NH8;(U?BgzjZJ|dGJ&87-};s|#Q|R- zA2${_gYby%t&5r8ZjokLj@g@Vju6N)AId@{dsZ_&3e};XF<6GHFWSH*vj7>PD9SR% zm@Z*E?vtDhbjz4%`U50fOJ>7yOq8#kQUwh0k4(>IuOmHJMGj++A(iJsc(2DH)**z% z>k-l(LcutdHg2pdnM3IM$0;}A$x7gBP~S*LgHwc)?@l&P5J2kb5=kbJW+47@H9|Ir zb%Q5!QhB!6ShKYpMk$Pv+bvu(hmYX%zpNS3aGR#J!qHqn=9MphhF*qX5hsw67jFX7 zXR(*3EAJSv;V_Im!@gw#{Xe!Ta}gpKo2K(ZHYrqyT@fp_4%n5W&tzY-GQ}{NBsdKV z4=r?HU{F`fotbk3QDke%JeBA`o4eV6S< zFb3WW-HvU77gUSv7b$U7ye;h?5lV?+NeSkq7}ekQOWdS{qdjLsgO&#%{!0E`G=TZ# z$lXZXbI?&Vk1=6oQbc>%n$bP6?-NR*HGwqqNqhyLJ9mIUD6`N5P7*bU8XKi^mAfhu zD44JD9VSd|iWeyMj`iP0MytL0nAb8)z!S`X!`eNE&O~w@Mb=>&atMtK7L~-;A1P87 zZ+mXj;yYThqJoVLZxn(-^L ze1Lx-0*~k48eQNXMSPMkWqgH++V^_5=3fIukoS3~lI+pld}jK|Urfqd_pkRnJ?dNRvf!}@wSuK0Zg>!WJ^UR&jo`ujCd3@aXQ0Q*KPlBh^94BG$FCs9QE`P?S2fp z>HsXo{uaf3tU}rI`*v>g4efmCfP31>P+BmKJ9o}Z-`YsVnUTO(1Vc_?hfs0e3!RZj zL1r;0LI}tSLnw4TlQXa@Pw8n*?aFC>cN4OFsu^;^*}f^L=%v)dW@2)zL1E5!$}53q z#FnJ?5JZ4wT{QA;$a;Ahbb`@?&eRwh*W10uUKLcfmnn}(WN9IYt;YB*($t>jt`SYp zddzQJQnau#)1w>$zd!9Tuw<;MC%Up4ec4IjRtYm=7Q`hgI+Ai%W5(ISo@v2}P5lM| zCo>Do326~3a8$Ig9#_oxR_J0~Zpz@jj8b5Ufk{}YJmA)rPMQK5(O_;)_y*25VsmK?pQIdh@Ot82xfOUA7 zI^#Kl&V_gBLm|T)9>i40~E`AKgo)E70#}ROScan3Vxkri<&!gK{>&W zoXnx}kUU)*#H-cF7)sb!jMau3qPw4(10Q^7$zpQyYeYb-gFry2}S~&UzO2+eY z4njVPJGWn>CM*C4fMO-piIrdh^wpH}_^$Rf5Xc)OW3JNFn-u=Ni;eluu`3!25Pv=N z+j+npz`GIuwdMikY8B3?;30yYn%}6$8>u7 z%0zdQXv(7V728~;p+oW0T96b+Z`qxQW|AO)*Q=r{a3?0tt?3YQ&~}kCFWa>2G?7dv z&l4uO`gEZ*W)TyD77LX<50Y3)IXokWD&?K-7No<`!ew4Vqo4x91-(KuDd)9tp-ReY z;+p*&qeinNr>YnjKbxs!+_eHf$=a~{MSxVvM%!RCNlm{Zh&J13D<Y})tOAvr~{G-l%@lRUd^+{D!sjK*jyheM!k8!AhrVYD5w z{T7Ij?b`@}JlSfrun}XGVh0jgf1NTgxHx1MABMb6&6uOP2pH_X){Ip72BcJxpi6SM(1TS~T`XE7^M(9*byqhlUd6!(Y=U^-_s zqZ1_C?hu=83*Jz+%;?hRvk?Q+C+KG-L=MbTDz->sb{_aB;sxV=R?~XR5FN z%%FWby1ba4jwv?=83~p4E=4M6Bp|g@GaiE(6tB?sIXo7Kn_`~bdMY(dq_G?-eUdRH zk~Enm%I(5q>6|TQ$zM~~ZY~$>UH)55SH6U!jo^C9Kh#pz`g1a_KzBfH7H%3CfKT`0 z9v&~rQT4o_Qwb`691WVXv$~#NzV6iO^E_*n_8+q1*qXhgoq}IlS0T8d^tm&{N9&}% z8GJMFK)KmkB7n_HSx?H-wve_XB)JupbEU^MmU1H*TQ+jjBFDiguMSii^8<_Vc)6S4 zydhnJZ=*S7Gs&n%JQHO>!PWxNWrA$rl@F`hV0n2B#*|r-#iB2ulTARjvcWFsEHzQ$ zA|yZZ0yl$_N17np@||AN6nTHRl^E01JDL}T+)i~5%g2ITV3$ZXnE-DAWknX%nOY7( zUL5CaVay3Ln#+4mjtX9a*xi<=REK%Oj)G~q(?$mwHTjrj#_~1KW9uiQ&Ft_c7!Qoe zz(8Lx!ZNrl8ARd7b6^N$*D1|G;j5;Fxd=D66#V$TvAE@VnxF!N7cnbBwMl~D$=f6! z!aRz31kMc(h}C#~93a5% z84b!s&W3mop~5q`u)ywwgGP$H=J|K7-s|tN<>!BUZ~JLA|LgYl?nC~sd--*Zml!TF zT=CU&U?Ts(0{6Fr*ROhGUd(7I87;}w$ObQ2luBmmf))uH?(3Q`@Q^gY#ji5sXLCr(y3NhCI@g$g8+7kq`Utu~_bW*tLH8Ci2*7>6atI80ud)b? z_=cBE41+G|$@v!ZEW-(dC%ZnBTj*SFm3mZ&`Amu8&T>jpaIqck48lQ6kJ^K^*Nclq zub5}b7R3v28FKrw-q_KU3zm5opV>G=^6ov1WtVnHI+DZK+3o~gUCIlls4^pKC(9w( z9_;Mxs2A#b)$NGd{N~Tvd9p9+Hn)14WlC6cLTg^YaIAsOuH)paqMXVo-V99ZA?cWZ zo4!DKN^{4EV!BQ>E<)XB2td=&=2?W30-TxMPJQ|aErW`U7Meg+-rDpQW$*YPA zRxd57d!rg^;rO6R!juadu@=8mtDc60fn9C995FKEF*=zhwPa)l%>rDUPVS()VG^i%VL>z%5vXwWk0-I+mXsZ&mD8 zeGYaJR2ORqeUC+#)ESrC0B!{1U~Boi3X6yyFc_`eS4*T39RnxWVspNN(ha zxf(mlM=Qx}$ko@x4?2Oaz|f7Rkr_86zEO-h5e9FZ&Iy> z{5c4WJgD!_+j}BOR8*H-GDeLm83M%$_275tlN30F>kB(3siD58cM@KEz84or{sHbE z&-K7BtBK33gJaRz8+{5WMDzQuA!^o+4qnpcN|_?4x8xQCdiOpc#vMlwi8W}jUY#i; zW6TMb<8~ITq>~}J7=+v5Zcnw_vF~+Z8S<`E`M3j!h+4*SQ?vDxf4v`8?KVLQ?O7T|xwJ!7iZlV;=Y3 z^4?GK{!lt$#i)(uC3t147RNlsiTuPb*##@MzJS*o%CsQo(fER9ktC=X+09g79OgV} za(1eD=_Q#cCDDdlAO?e>mYOK^Wdfd2vSt+c7Oi-9=H@w(S)V!WgeOJ*Y=W9zhCRz$ z9tH#ENZx;#l1Y*vCnbzk3`?G_kPp*^CU-?LnU)R1$yr!~Tcbi;oeyX}B!eIT`E7;R zu2U+Z_+Cr%g_4WeCWd0`GD0Z3e>3=I5D>B!@lwqi$QODY$(whc*<>y#gu1rIH7VFN zNR};%)JU=P>n4JQ)sWslID-5m|JJns+62%arYy}_A;L0W*A%?M{`+))r<(tDcjwtd z{{MUVb#Vf8esJ@jC*m%sMa!5LF_^ARwKGs@^ zA?Y0ceNNMkkF6FkZ0${FjnUfpg3ofvpLf374!5872AxXf1*c+I_?M*AWy)9n1Tp zcD%Vr0~kNZv1hE9fQj(3NJ{1cZu?h`8_P~eYlmuXZpSq|yU1;-r_=*OK!qsO?3=0? zHeA)p+Zg7fdTtD-b<+;1LVj$Daq{eys$3)+DlafBjpU9(K7V%2tBv_Pocf(wwm`=$ z+|27hqzxIJ%3^KUN+54_Nh^~JfF+>5Xh;LXi;7yp41hePldZ%?Hgq8MVojZ)8`g)} z+-$lGY9C@7hwyjW+v@exEAM=#j8}gW;h#h(E~4R|;!a~>DzC`sW~E836c{upn`TG+ z%~YX0XeVslVQz)8>eXlX<3EX>LA_n*`UbNJAe@qP=Jn><+a7`Ps|JGt`puef>YATD zrEu?|_V+%aPn}bH{*CC-JikVrxVDBKK?q6xqCp4CWv6$I#a7i}?*m(YIoNfORjbW= zV_a3rbv9ogjm0_ZOkxkng-4O^1AQ&kJbO^WunC7N3Xa|omums%lm<1I2NKkiS8#{4 z(MY&&LW`3}bAc9Im*;6#?e=Vi$kH+TwtrYP58EE~OBFiqce0PEv+C|H6!q@(iGcGmg|)q479ZvbGO z4{8*dO$dtbvx@LqmJ3DQWfqStiUZmc%!l#BeD%Tmv& zc8%d2Hzu=nktDa`v2)HAw}G&dP>po7bJSgRs{30_gM4)8`QOD7G?V}4@&2jTV7(If z@}H`few~&63PpCWZU3wIqlg})U0NC#x+4C2@7Z3}{pv<=lWkX5#e61cZ==#Lz2ZwHu;cQH-M$?+L$23$=*;!N6lccvgj6z`kmb`3uemWQROCAMg1bO|DRZ@0wMyiFtQ{5I?ee7~|C3Do6IL1{=+#ecYXc|8{;;N~$t z1s`1<($gceBpZ@}3FVp5k3+J*x3{|&5Hc2c$J|kO1*7rLS-Lpk970*p<;$)+=5SV> z^~ht4aJsYgRBO>AkD1CE{a8QHX|3f0i1cLfPwJ5lv!>RD^#8^&U%9;r=l zjqyl24aDi`E1MKr=peqv@-gPIVu<)c90gItwL# zH%8<)=DJnL(k_Gz2s5db)hAwbSF5~fzG@h>f;NzF{Fn@9K8+5zwYjvw?smIeKK5E( zk!s%_uwJxC3#^{c^19%@fLmR30$D|PDUz2kB6Mmu^`U1?G z>r1b-ou$}Bh_E@Hrzy%{+A@QmR9-rKlc7-8>8^$r6Te&QxhWXYKvbD}YZO2T6u-7^ z&_`(FiOA>A!`5acomhBn1d(}QeNL+1tETSPN+A}y-nec@%x_MMi4)A;BfZ}E?{W70 z(T1v7x9PnDN0r{ZNS{C2I6nFLfBx;mFDI`yz5W|^d!_h+JG|^vz3$OCE@ZzypS;~% z2D1DHIMSZH5}wytPd&bygv2XPY{Ps*MZs{u$Ptjf;wgGWOkDyLIe%O^JAp-0PCBQI7SS}xCQh~&xKZ?F zr7)-)w_T^!i$v0pJo@zF@Z{}>ACF!get2{6`tWnTAq4$B`t%=%e_PrN=_T}dQ0voa zZQq^#{L9IU!{?7S#z_{dtqm~uNe{ee#sm^zlTIBkLzw-7&CzFLX-b-rk1vHWw_$aYkf=aq_ylIqrqZrft57pz#VH~&#~)W8pU1ZaKW?54UU zY$~r;6KGYEwLWw&4(<|rCev6Wkd<9g2l84{L}gKK)@Or=QOD>FyKa&{)@Fs#uGPSx z-GN0>gK}qf#qFI&McZ}x>KB`gNk8m)d<(++&!z}`haVbpu4;|4E1%)e`_E_ zn>AJXAL5G(>zW21e*Xf$o7;bpF0S_eUFrY3`}AqG{?p$6_Cx)rd->hW{_Eahn1u|a z`DIb3_u>9q3u0eoL-gb1|M1J9dGqmymCEDQjqV-|la<+VsmHy@u=CH9XfExzInVJE`Dnhktt&5Ht5yaR~I#> zb;i{wARASXEr`4Guo0~~dSq&p4xf`whg9;F$<1>lNGHiIXqq_rKoxE`$#uU7^?)kX zOT$h#O_qc~RPrgAJ7z=YJ$(To%%-+>>ln}f1jTLYb;r~Q&m~*lAsZ$CzXi|QXS3;G z&rREB=JH(j;qehI7+I10=5E-Ce~2)xRz^Y>;9ZB0Si@C6O+?AON|9^$=daCXxB%4o z3|%E*Ns?28pG{}Gtj&Bp{J#3{+Wa4oMq09-s_)*5ig<+UI%M%VbPcwn8=NM#)c0s5`EbUw6s>uL76XL|&p! ze>z87(_bn@@`o+*h zI)&ue7Nz}J7o~-dMr_Or)6TE57Q)i4oZDAQP51D-hu_WVe=c%;2i%VS5B3Lj`oI5R z|GAgnm!SXGvFmqT0j!mthd)t4I=UENAvPBA((lWVV;3Vxlr&%0yNya8vQg&)`c5 z63H19=3`+HzC8VKTJnMgUGf6TrjTD>oRT;xL=aArvJd}7@Plyld(nsg>Ic(FU;an` zDK4_U?Px?JN%6oTJ9rX`%RG1zj_7&tBrIq0-@HgB!IS?Cy5v_{Bz!K&(aS>-gn7Y# zV^JA|Nz7;;n-~1IAiNL}k6Hh#AMQ1NYo7m;!-JQv55rk}lQEW`|DFBq-JQz$AMETt z-2eCT>(+{U2f=}^399cmDzKWre*8`sLw>&jAqvBOf0C5b`3S@}T|-zm+Qiz|f|Y{I zbH6S%)T1_~mC==iX0cnOTDGYbU*;L1UWstL-oqd`LY7pJch!k41y*hPvS6$~nf7u? z@cNqa^{ZWNg1W9?v}ASIV73v>Of<-pB?UO>Jr*QWCBfw+(Jl>jNy}*s4M>PF@anplu+%RLR88UHN~72putd;bZ7J&D_SxE z+U_%*Ec$H4bmfWR^U?{p!pK=cHjAoQPG`(sWTm0l1B~E z8_Q#>w$3}}ps5n@<)XioDlYwElPF#y41#W#{9-HOa~J;?+u_dMpM&7Eq(v!TCt2YGa;(A&EKq)CnV3hSAMq?=c`5qZ->aYVxTpW87bI|U$7Xr zh(w74$HDPY7z77`P$@>u({hVEd7^}*`Ld}!c>+a90qx_0iK$U1FVcT~a})5Ijm? zXX0IvEu|6p=fg7f?kP`oJp_XcG(}$u?Zp8w%h|ZO5~%$`rN; zE2D5Tl$bKOQtXo;6IK#Y&ZBdqHvq`CNRBW^JQGlje98>9k;f9FM=yc8X=FgZ-i%ek zD0$!JJd@iGNW-$dTyoSpZy{FH0Z3D|cN(@AW{5{-0kEc*m(mB5mS zutK`AyceM)87n2pR5x0c!&UQ;fFS>>7ZaIWzJMd>hDx8vG0f9IU{{f*Bp^opEyI%f*iKdY)kOHut&E!ldgZuZqWgMQ@UmuK}3*ZDKsw=NhILnr($&1O#F62Op zN3of=0s*^GHJeIKVbA7`qM1vxZ`;Sj|cBKfd)9V-gS9L<3c6fBDwSbUYG z3%KhsIcssPIs@UAm4v1|n}9$EHybckTaw#~m}C$W;qarl2jN59PU;py{Hw(>N`Mz? zr=lW6j@5THCs*X`(`SXV8gTBJ^%Nn13a&{(2-Ib{WNP9m;FJ|Crtm{xTuJ-H{7aT4iW5=zzzhXtZPPs>DJCRQz^uqfoY z%Xq9=9QEE&4FJO|2}qhmu-d57z9MJe?(TnEorlVWDCc5)Vm%&5w=m2OADbJ(xLLYc zSbTrn2 zraX3AwG*Y3lXX1|Y>$c>M%|oN`{Lw@8aFF|6ghQ9oC`eH)d3+_t;LJh`e>aIi}mnI z4B@P+lu`boLTUj!3NF{(wFjaMWdde z(nWn?=p-m5)LkUEEpDu9?8K#|a@T_QoGpk>gA04vYS0(gyVB%jZZzxb9>3M4eEp5) zYRViGfQM0n%9Fdpf^knmeqrgpYp`vsaYD2o)9L9e6WvXsDT~fmY;%={4#iJvK~fyO zWp^T)NrC`guZpg~otQYcrbEa<+eOa2Y}2ySL^7Q`PnhKD(}mKQMN9}B+#@c4>73DwPLOQ7Lu|G!cthDTqf4L9Mhr}! zpr4fxIWSMD*dmG9dEldn7mWK^P3z_Nu*RXPW7Q=vgZAm@@?v^Arra20BvjhF6se$* zfYeUScnoGxyh7XO@K_*jig|YHsnj%)#&W3iNyd~&(qxt>w+oM@bGDcze@$Jxxm>V! z`ENB{`4Wycg6k>&P)k|s&&jv~-2u5-xM^SjKHZCZc)TP>)$@W*C8+#yG-%4s>Uw_p zx>Kvq^Q=|cf5?hsYxa(I3Vvx_h2VnH=gtrxt&{p@@Xf#jPfOq2x$TMI;& z39^A#KCEhk<>fUPQ)WpPi@tzPHUZhn2D_lM)I^Dkko?FC+zd({X@YFacX~-vaXah$V-F(=GuF7G)xDtHNEcUzuP z9p(u;3Z~^w8y#fSnRlEx7&Vpu^Mm8=j?LJ|b0j9HIY>&$%zKCia3O2&@7gOUHp5_Mag&=+x*wRvmz!GMzl!aXT-FM~ z^q1a5fPy9VZwIem^~SuI(NZ$plDU-)Ua}~aO!WmV5_I6GYed0Q*5nijg8;L5LJn`t z)j1?FkIq?vx%_Q%&Y|{H?az{;;023&tptpaZpB*%h9M=Lkjp6-R?tWU`SgkC{96}) z5*-pEn{#098kKyPPKM+HZyiGNd7A1n<45B+ygV+L!0c*qGGU?|l1@j>&P|$1-n|dH z=Bn(#`=0EqhxVfm&}I!gbTXc&Y3*UqC2x$%il;G|CIV8Jf|^*~I#$9hc$$uAbRO6x zcb)b@)KVIql9TO0P_T$uH|yM(7PN{yX7L+nNU$B;^YO!jI3(h z6SKx0s94!0Z#kK=GzXI_EfYuqfHgJ=FHIn#P`ZJ+(18rqVm&r%K*WaIoB(G%q&Wt&@s``;K@9sIu|j2Q;*1V6wbElRpAS#rBxE?fO- zyY(@3L_(64G@Lms1e_70#6`44LTk=>%nw1Xy?IEU7uSusvTG?|5*^ zE=Dv`?r!9-QLNjfB0aRJ_!Hb4=) z1GSsLXkPjsfJ;{xNH9)*ICud8(3rCkK(+M*x?NC*vFsu#cm{f5OZ2FK6~)_<AtvJ%Qwm29Gnltf zFxU^Tlm^SOY(!~7XfXlrUg5_-)Ccdn8iOGg{O(0bE;$Y{Pf8}XNUxXjNl)G$7)1l0 zY1j9htJqCiOnlQanX;mpc^^rv{NArX)13=Cktk*#%Yq&h6WsY~8I`8c7{rwqCtin? z7|S#^hhrkF>J8&HERMJ`1p+x!-sv01Jl?G4Q^)R3*w{{_A+DvQ;vAw|5YN4w$33H1 zHNh_Mh(ruqs61~e!3M*lhs9F&Xh5@bReZymBL>))l zyD`s2Sui?-Z#Dy+3-Oz6NC@wD3r_Mnp4fCDKTzLcZgYtt_}0=C475GwMG4A1^mx#Z zjB3sp(z41Ba@o{$nREPCM&<&e)(R$X2$rFfvFun)RI?g3 zri!9X^Tn+Xxsn8|SlmhBGex_-)mHnq;yOjenbCAkqvd zqt?K0DdPczohi+8x*($hLZ;|g?yAQY<&Q(t?JUvB=evYN`5cN3%=Cj9n*o8`fBHHJ zaGprv)m5oJm<3+z-JRD-5U8!-k$lC-plGXdA3?Ey+}7Vd`*uI@vuvtI^0tBp-d46r zLNUMR&)Gqm9%Usf#uT=e8n2?AAr5B_j)-8x8IlepdX?8l{&G&!(~?H#*uHjW$u>XI zS&}ADh^%vTe6iozBJ%%FJMY~F3wEhbvc>Wr zq;?Yr7SR$KB^|fo)Y5*F+PGHftD~y3+icDHiM?lY9;>{xz_-URfy|?^ZhqPB3T@gt z;W|mSP860(XX|H_R`(|%0$k+{g(hS~MG_s%p-3tsqT<(!m@HF)v|L{|a2@Kfg(3On z+xNhNW)4lZ&#(9f*E<-mAQmQ-0OO$8P(J0~KN zY=PW*fGX=IAlw{{7dzqZUN}I;VZ<+(2Rnt{Uso!-$ibY8h(M;L5lbbXqPTdXw#P}s zn?#bA3z{0%52?r7WXAX$1UuoZfTl7!y}wn#k^}oseJNGYd7=pRhPw1%H@7DH>|mXL zSIO%v*(vajAbctZDWDrY03f6 zItn%pzu+Cvc49*dEP$aB{kU^bVF5X?4h1sbg3nlmAJx%-1?ZkGSg|0fJQixlqY7k< zV#cAq0&QsG#`a(^`12+iv1tNXqg0v;&2gY|36nf($P%G_puz1zktn5(5SIz&t3$t- zO5|+GosPDkb4dis1l0)%6Dr|}>A;J;2d{Rmhp6BR!V%+SJZ7MK!M%y_|0sc08I-mZ zvP^!8ksovg-+@O)%CiY8l*Vk9Or|9Y*5fgYO4z*r&nxoxIYNzh9mcRxHF}6OBwnvz zeY(VkDq62I&jCwvHToXWx&ki!8YezpkaO;RD3R81dL*mg`CI!uk@ zg=4B)%D;7q25?ApQYL+hgk*+Qt7D%vB4t4{F=mC(R7Q!nj>h$v$2MLS?}D(E&B9Tf zOW3_xTaRhey^1{ z*WA(3NoLR=G)t}2`82a}OHx*_p)L&LB7g@ceh7>fpI>82@G-Y~CttDzy_3p^lTG>_ z3G16c{uW+eN{0{y1AB9>V})s~qyQBX>f+1n5NAc?p5_ z!UWoDF2Yzy8dlbmfN`uNB?A(w{cJ5SWd&;`%7Fe)@)$OIfW`BF|JVP~$;1>P)AFUx z%@H;G83AtMn_(?qDW zWb*F7jOLa_RAXs&!5POy+i;x`@u#1Dxl9Yf({vn=LW z24yHVexwQ5?@s4Y#8}MYO?}V~vpSRAaPcflv!d! z`IIV7V4S2SD^3tFX9holM*2jhs&JA9*`=-~AV4KH;EL>i{AluAh(^SX{Q+DqO9hl- zPx6f0#9QjJFX&9T)Ie!&kfx&+5b}UYhC%5wRAN2DYBbjVp!1T?sJ!>nbRqd$VHEyG zHKe4 z>4)F==mRigM#fm5A8}mHz;C3i@Ff*4Ma}dBK5o^Rb;)a2XSN3BlWf9-Wv7}|ZkiMN zlIp99-b#_zFQm1qsH@cFbw|o7ePC{QtNXgDcF=CDsk#gJ8WdI6fOKs&)m?zvsHN(5 z$&0xt`7HSz1uG1-I4vbhJXw$gBF_W=>CjlsPB@ew`UPGG1pJZA-?}*D$6qo-M22KL z2&(4RFKBH2lBU)#YiRu(k8+9k?lx--x=Q)4vU8#J3(J)^Aq!LIxu@Ld3`O}>!) zQc)7jIb5;hIeI|*Nzf(7+D!y=VhzbbdPx@;UnMuf=GD)cWK`w_yL!VeSP`gUw2d>H zr>U|0s@U0~(=WPfjO z_o+PO^X#DU*&Cjna9$b&!NXq!D|EY{)inj2Fla0Zob2MqBxUGc=wLf?3JK}`NaNT# z%ANl0^uybu*M~t+j3`v{>~^gSi9-4R01^D;hl3aP90p3CUs>Q>v?0|64IU?$kPleG}ElAu7Z3cSe;1Y zLd*S8S!nslS!`(0xY)-9kDSHsHsN(`oHot?J)~UW0MEhccRI8EP|ZJ2flw*m;9J~1fmU=NRjeO@|XWO zGCKAD|657OiyADFg?>3sqAnm!pOuP8$h<^-J|H`;xyu47tATQrE z=)>Kqwgr74(&Eh3ig)qu0)495O+B(5?u5JH(?BAC7SWWMH<{RWGp_bxKBGw%DlcZ$ zXtoIBOoaGtDCXmF@-gUY&tKCOc>85BFNNB~u%(Bjv(xdKus9O~|LnU5Ps&!t8bc_V zE=EOS3erd#;I@`q+24#@45f%L|A_pL|5DCiNa!ryf9kU36Db^!oD~U22@@WS=PALQ z1nM4A?g*Y)ZFc}A9p`C^SE!@I4V;hvB0D^W0Zv)Ovsh?G+{xI0wbdDzt%u|dSCk92 zPL_HcLNVP+TO?;KO#`i=lrZY7Xil0YXOc51MNOFT3J;dvyXdYRv(bEl|Lr-kko}lp z&#Wh9NuGf>_ZVf9zKfmLIOs8|2pi)MBp+b{D1_^%P0J~YvGrL*~lEXg+$&;Q?zg3hi|&hx8re6=yX5?2?S@jKf4 z{ry#TF=PMhf5m^!C>6MF|Df)-1piYNu zywsBUOO1}7ix(hM*(XH2owM@l#grB=VQsE{NO^RQ|Hd(pt-|S;f2;Xu z!4XA@D?CZgS1vOqIk~F}BX{NVaP@6BP9_jhhS!x!co@p1u`es(f8XJo*EF|F(lQ6p z7(e(OK3@KHBvIfveqf#zlhDdack(i+mZ(^I#)Jy;0j-RaWJqp!;Ise#%9McGhApFW zKtixp(Ib$Op4U>q$Ri+cVSl$vQ8}VTZG&eQNu1DL98G368WdI;I$lQ=Y~{vKt@~{R=Bm$1VT(iP2d1OzQsg3bdCjr3=FtLAbHcD{aeXVh(OD z&XC*k)qB?D>5z$Qh zRrmrzl+Q7FIG6-f9AY7Q18iapSH1y7TSK-7+J54WkYuYz zNV4mU0FTf%XQQlq^v;qjnbFiS;OexRIw_wj#wi7HN&5hzUylz68u2g}b6Lur4O;

2d&djJTJ7-Bu&Ol z)gtC`qk@-oejbk@P!0*3{O1WnBR~h$dmTwkI@M$8)_oS4mRiyu7*4(Ro5GtOJR==I z1t_o#3#k(UO2p;)mQgW6aZJf*g(aO}H37_WD6wQRoGT|A%&lir1a&$O>4EO?cZ2*t zCT~ms&ChQSPs8$KxqOTj`QLW;pYB)k|2%!R_mKbXUVhsl`3u8a+|g2jui9v#-TdTh zfw@npWW|C{pL&p%S{F?0yisA!9aEM{hu}a(3rL-#B*ZsAzdZbK{PW4%=Z`kd=OY%C zDO5=9^{hwtr%%RwO**aFh)nwq3CZWrq{qo`f@e7`r_Vc|LXoowVvfbT!TV4LVt?*v z&9zu_7;RChvz(BzeDV6tKN@}s*`<%Zsk4l`E#fB0D1;Z=g4(_O+-@a1uzBgmY`n;c|bZe zhx~Sw0Y@TJHtmghaY>8#4iNfCuNPUhE3#PrQI7#Jh0(&L+ z_M?m)^pxMNaAFV=$r_^L^=Z%}hnbictXjO)i**u8DpsD`Seo}DsnxyPvOdZc>c4|7`D>EJ{8ErpRR2k;)qfJ` zr`ml3v}rVsjjFALWc-p{0pmEbZd-wDGGbCV#m8>_09C~uf>KCiBc{+Ot$fJ1U~Z4W zlfI^;8o}@u&y~|V;#c|2bQD6sd@pYV?K%M8$_Lue-LY12`Pf>zJBNRt)3oDbs|5@l zvgoZb3g<5{jTz~Dw;gUj?F~94eBt<4hW}yf41|A4N*$ss`x&v4hHbtGhWU^!mo{oz zb-BdkBAE@rgJqngO!6krXYxQtQ#K@kb=U}pA9Gd6-p@4#89oEm&SGT+GtslnVHc@P zp};rjJ?&=dBv+&?=2=Accj2GO>=fKZNT=7S-?sfH!3$lxRgWMqjRlcwiD7s&&k2So z|CM3i(7u!-$Vmw@ViAQ}h?4-(7BAnjCYXWCUXb;lVdVS%b>+iSbe{{W5aAx5aznPBNu{*YH+qT`Y%`dj? zq+{E*ZKq>9={Voyob#WlnyQ(beX}oi)qZQ^U2FZG$E$NrOP`Xj;3o%q$xtqrG>uSU&6*v`QnB@e1GYa+v{rTM-oD;l z%4;xeGXWQNQ^mSa^QYg1r#JN1_w2MPqb%dRj*s?#ajJExANDS(+926CzfktQ2qsp^ z$0dSD+a8m#5WB7Ii>N2OVmO6dKV4U=^p^CvM1YSPwNqY>_EL~FY}Oq4hjO?P&sibV z_=Qe)c2y%0BzCqfuDR`!4wTjSC52x~Dxs`ya}5eR8v>mMa23_{7$R+EQC zXm;`K*nfdG&4*Jh-{hCrj`HI_7=tgT2;WC3-iD>GeaBmwaNH}jNqe1_*JAYodcnKUYfg{|Nc+)%$OXP-vW0)Pfq^=V()^_N`OYt z;}fCl&jhf?j?I(Ye=~SCNwp0UD{DWWyD(SGr4eQ8d$e~L&Ga3l2ZV}mJ+;YWSC(Q% zz}yGcp=B2<7UrHzg^D$8+j_0=lhv?`n*%y&clcs%Ye$oZiPAiD$zerw@?f|C ztDCT;ZAfYm>bYg+r2U*bR*9e8zJ#q5s4U`Q#9LY*sJgDm4DEYVn^l-w$Z+TM3qF-n zg7f-0u9Q0a2X;z4Ta@hVuOG9j;+^OW1s;kYb4Yn5gMht$TIYk;N)+M|`MCH{V|Z!| zX2=6}Ip4Y{OAfdE<&9~f?&0N0{UHPoOJ?uj0Qn7Sj$5|R8~m1TT$?c(>|zUpuV2$e zzId0{rk9=9#Udp|!X_J1#>2}+5@i;nGaYqv)vGTubyO>NLc^?_HK-j zg=U8Ok4_9dMG~qwewze|^mNCUY+hz%+pEb5u2K8wwu{$?&DCR~mU_R>mI%KU|2S8| zDsj>+%yi{12Z0tMm}^eEd|_8kU{uPMmfV3C#__wq(vfV&`Jr6XlH+t$k-H%CyFN7Y0d%ItJ}@)A225OltCCCEj0z|Or# z1*aQ!WPQr-+ZriYT(8WJ&N!N)S0wK0E7m|7rKO%&nvc8kMc6<<2s(M^52BhQK^@_N z4CNY!asUp>`kKFHN`^8=3 z5s+-^m?(!+8=&?GaQ9v<1$GAYH+Dn40SI~ljBh|NJCYlmHWB6PEvnUj%W#M60!Aj1 zMI{#3iDwYxq@(TI73z|&ce-^VDV@?S<^Jszt*iKl<_#PkLH4KbR=* zi3y&A@z2GcT#+E}wVM~}vi3yQ;_-o!X71>F9SO0GxyF)WuAs}rf$?ckM(Yh4bLI>A z(ra@QsNL1f=#qx*Yqf>Bdh_};m)+}&rKQ^L?OMAvJZ+Af(SZv|Vn!`c>NnZB11 zzvBSm@ZLhZ*2*bLuW=Zy6iM{!F5<#r(8XGzjUNsDQS1syX4`ZpGwi;rWMb?kyS~5<*mp=yB4PAME-&ugTk*p^%jo5yDtKBH1ALxs9W33;=Qxdz313W(7<=Qg4%^M#6i>33z*T)n*9%UZ;qey&C3!7i zFj(q){CBi4>Xaq%b_d4|ujXD9uyY_2cB;@e%lV9N!&8;9DIZ6=Fh`qr;W_ z9!J&gTN$yGak>JmNQc$_P$J<~{INbtmdg{ub)#o%U&A~6qT)zHPT25gu>vuL94BVE zUPl5;5E#_*l_l_A|F_ZT2%;YehU`)U(ENxSLr~Z)C3Za_w;Mj4s#jBGcw$p8hHnJh zwCCK()fn}LU3XmT++rb?;1l=Bb-hdz(7+EqnTG$0d7~2EYrkVc_oIxTCLeH|Z|M3& z)?fMM-XXag-{tGrbkLC%}xj8l7YBN@mLp0uANAN`TRfK_yh&7%4bohiFK_r{x>Yza!dZK`A6b98L2IM7d# ziP(^Pevly8;wbI^e1CFm(0#o71;SU}HU3@;<1Cl!PiJBC1Z$Hk-xy{%2}=8f)g|Jy zi&Q)jX%(F2oQ zzMm&QQ+2V$ySQ4dsJu{h8K2>pWF6;A%h*V-7rw@Vdj6BMSTbpaPyx4Dw^B)#MQ-a& z7M2#Y*D>QgOeqf7XyVfFbuO-NknV?7v0s21q;Aj`Bil-ito{?^Nk0)dzy46-Vuv14 zV(^YLx60xiN+Gau5y+c-uZE!z_W(zHcay8 zhi6-XqZGyA!wCRWrK7R6w}MH&LIs`H%4-d9+P>0Pt6veLefXq0(5%x`;!}gmkfkiGI?7pu5fp*(Uw;;fkR6v`G*U2w*h(#ktgG{vI7`P=tm;byyJb=9)v zlqD^;EfRcNTCFWs+5+3^umn&n z7!(id@m(tr*>-Syt2!09eOpx)x=^wz=ksCC%V8Y-)_Rj|KmA`a5Oe4lJi)MPO^gk6 z4C-NHR(}Ep=BW>SI#wk=Y9Bh)KH|RN%;aC@j2=*q*#uEIkkXSAjTjwQ%u7Snos+ktzYcNDFM-jRI2l z>f`|7U2(X^rmlGVDn1Z-g09YAcky|BIf2i2m0Lh7F(tVDhEsZmd>pd%qST4KnDD~2_*tCFyS!D4pL)~EH zyeEbhqE}jdlF^d5TDLIP(xgumPf8s6WDJ)PN8%fl%#BEjN`{mX>mv^SKz0%N(Uu(sD+|G)rW9aKFD?@qL!*Unawat+5*WOrU2u+^RG6N%!BtLKkYZX@1qS4#xevDptVbmX9JZk|Vb&7-NGCbp!7eNLVg zY|RW8c1_aJiKquIbcM!ne!O6zOpaSZvn14sr-s*V_W0Yb_i-b4*mRmd)@A2zultmI zT?uZJEvKRF@V7@_^!2$$_iy-n5INg^%{#*O_uiP8)&W11i)Q>Z6t1miMc&@w&ZG15 zww9q)a_{zSnN13cSXNCnJsnFAUVjDR$js01;6T|www{)ppzxgB>BXoq8s;ZXO^@yI zcExpvH$TZ|5|(#&pz}_oHPJrc#R&{P!d}#!-&S8a{I}Ux7uvH*R+OW?Q}|rJNTE8< zZ{(;fSTlA3s{Q%}*_wX?3M>4aD#IK*ZwbtGkG9OaU4Qp~UHe#SNtp;!B{|1LopHvE=?x+p%Ke7*os&u2`w zeN7<3$k$d<;ZPx0iRF-VR+D4aRwEpb`>gJHG_Du^MP1*OH}J{yuGs)T_weHTx1&1A zJR!WJ^}Z>i6~yOaOp`o%r%v6l|&L;8L9>l>5x z-q;@mobCSjVtfSJIZy1)vpG<-_FXMb?7H!@?{#(&Yb*Ec%76NKlUxtdJ(Bsg76T{w z1nyb`zixQ9D^~y;xHbYy-?_D2g2be#sCbIT_=gZ_t%y<6sPLSOfRPX8&LS}% z2Om>`L~o&FS`{S{2Ib7DDIL5MA_5N~PwHj(V-sJa6Vt%EY{=1-)e?!{QtYJM&6m-B zzY;&YySMqMeor@o-9;HrD6f)@R{8H|9wo0U+A2G(qhQwj?P@AR4~BIbUtkUXS)|HI z8mC~|3bnq^lrCUEw*FCw|M!X&`5YdJI(UdfUFE3$vdYkuKG8<_=lw^1`duJeQ?@ix z2`foy*qp|@<^!+7dbAmtT#Dj3yYF}So>f9O=%#GG<19zWWZc0h!;Wh?;*x?jR0c5( z-6j^oN${DgWJVO6$VO^nNI2Q1gkw~IW7o0reHSJ(W8O{Ou=vZqw*-iL$+rajzN_3s z+O~fP8CY%Ydo1m7gIzbRHNaHma#%<;0Fc6S781+ajX@2xiSu19gLd|MHPY@&iNd;Z zd`o6zHbd3vahIF8=r2wbju$va2HuF#SDdk=V+S8mfJ8`CaQTtdqgZS#ul zV{%8Qv9r>c+tV}|A9iU6d7G>Lo`1Xf2+(cn*?M)o&R-?;hyH-@*)sQ0{za~F;j zpw&~Jd+wEuzz}eIa&{g_zj&^Gj#SaQtUastPY^fGZj7u-kMjlNV<3mqsfh)?#NlB3 z4k6E~eTCX&oAgOrWuf^g-U;(;MP0UUyr)H_y^No43DC2BLw^da!1~V{*Z$T;u@LBu z_>^~Zw2BYp=#;wMy=+MUoFBhz%maW(e*leuzP8@=^ZZ^|HBIan4(q5l9ViY1Qq4<@ zz0gJh#i~7R@0;sEkaarH+bzisva#Bs-D;NkT9kM8N~oUC^r9-JOSzslHO}cvYw_mk z9vi{c%SmzP>1&!pOSh9d`Q|;1A?oTt8!5CUugwxNm7>XzmmP_T!I!c`YLplw?xL&h zFhpvd%`n6pJP&DaMEHpFGZ7RXI@r`wR?s-8xSFaTL(}BSa;I*Uc@A`gdJlqn-|ha;13F450Pla>uA&bqGR2x_r*BAYVIOGQCHFvUssN#&f|0ir)y&$}rPJbqE-P)c4uaIv z&L6hQyC3ZbzAMH7)8gw!;G9dQg$Md@If6u3S`eu1V^5KJQc$a#dt238WHZvJ;`WY} z#74n4YohJdY`sOnF*gEC&??5=0`4UjH<~8i1y*y6M7{&+TJ>UpYrY+=4-UZH(OQ{U zORWZC)S^RhjTBvZEphP&Qi!zkZ1EG90Z}ZCR1?Y8P2q}`zSE%b z8DpOEjhv^6(dBQgxSstZ;}RGL37@Ug@q%P}x&3JT=K@>o2m+{&r$rs)D!n81AXtg0>Byy%gUmK0v-LG)8zmO-u!`r_{ z7lRHMwLBw{552(0b@`)@n@dFedx5PDCog+Qqpt%Q12n}kVXDw)&-QWC(P$KMAR{tQ7$Y(F{TaU(G@ z22m_Jo3G z74NSFqVEPD57tH4n|s8>R+3f3cRq&}u-P#O{S5RY|D@YlIZ6HgMtrIV3(Z|6_SPk4 zuMMP2tfxcme(KapNdK#sO{%&ajsUSSOySGD9}j%=+J!PVH~(f#D`*jh5kgFI{c)7d zexQQz=Te1OoMY{VN4G-x+4t`@o2Q!?msA2i5V15m!TngzHup1K0`$ut=o26FmQ}}d z=q?g{>=LqVw6hLs(|1}2hCp773Qn#Vl&gDKZx z_ZkL{i95+`*wTj0c#bqE{}=DAeO`&Nzwp=MhYf8ABB4WMaWpK3mkNhNRd$zU`^@8m zlu$soMHWA+VXL;&!Ua{hRhG7Eub2Fq-BWajDaQw;XIbgq=>O>F>oIWrB$yLuU##Eg zb~?eQ4DcnzWtL{b2nU$x#%%SmxRm>T;cFecA7A0-$dfYjCzw@jl51uLWHvYBcyk9> z*Sm54KhBN9KBwcmavB0-$l%)kIx`nz%FaNyLid@%oQ$=oaKd(rB`b=VRS2{Y{8Od^ z=eOr8**SkkS?Flv;QW*@0oMfi3$d4m9dTT6$Z|jP%v*PA+okBmYYsu@Isamle6Iu$ zPpSegY*YGXQtZ)Qzd_#14Zz*c{O0{iB-!-o%F-bX8mh^HW>DCzbNX*nk_BGWsiRcoI(nOyM zuME_|s9w6KNH2f!aG3lzCR{ouZz-iQ{J4T{vi&Pub<2QVryhD6MAP zEItz1J=nFQE7$kGdx$QnmewtQ_7?G& zE-cU3oj}2c|HLfKn1rYmBk7zH!i$=kKI#c?=oK%B?w9UNjYAkB#IE2 zdI$wgb#i1n^kJ&C-4jJk^8?~P!HF!pYX9JCu>U2@+$Y=a@2fZ+lX`QhTw~Ib8{^Bt zpwCdXdnRU=TyoM~EYM)ZxZX2UCLI+qkmURnDlE=wlYIv#BC+$f!Yv5WGt_259pp12 zv!mf`NWsfbAjdg7k(_AneWuxcoD8i0iv9vz`dT}0u{WHaOd15paM8rnj%6e~)s^mV zElkX4yW>faadlE0No!%D6z)Y5V+SVlTE8y6`h6X}lC{nu_!Gj+h-8k9*Bpx&Pyd~_ zV8!_qZWRX=lrV6mBe1hxcnD7KYkuK(_Coa<;4@ z9_+`E-3#kqPXKKkTedwwZ1`t_SzwS4`xp&21Z{B_%wbe{9+>TG9Os|IlR<(VuO!yq z0;&sf3E>^kuz~c1Xr~!r6f&_$;`mqsWa^+?e0*QZdKqY)1Jw=c z3RJKK`fLOz5w3Q_2t{vjc;Sn(a;!r&9Nh+8kQ1BuR~$96=?a!`FEkU@UyVVZpP$=% zUs0HYIhYJ?kV1@qk{8Uu-h3F=1k&qH2L0GpM>hXFyS1HgrX&W8&i6ypj+;cK|Itix z)Em&0DW|4DCK_ue4V7OP8;bH_C$8C?f>}Dfu!$Ezk@)p8f!*2hyS0drk+ z04Nm1cAd^~>EdwZM&P+bCjj|w=X<<_FUYkyMYX;K@&mh@ciN{2fjw9|= zq%7-%xtjL$s2wVxk-6N-n};zksB!AQR#9cKH>ouHpXPS4UxH$hQ=S$c27Hc>%o^xe zjE}hBA~!UBGE*HX2@y-E%%!gO7lT=d3H$=uE2{C3H6*ckL{`_4-atdm`+C*#X+}BL z2m5DsVn2lRYb)>)hH8N}Ii=@elmQ=;mzT^a1%j*@&6~;D%PAd&*&)JkMNF#{sxq`z zp5I{H*@g)ZnMf*#%D-6QD*6k7%*1)G!RSJ3&Mp_oj!(T$nzyMthLwU03kG0?6wx7x zY9_y&PnQ^5oo_Eg#%DDTVt&RF3mtgbOee^7;E3`A9Dhwq z**sP%>`t@7mOObTqmk4l7I&5T&6~5hs#f1%&i*s#3m-*QhRr`jr;gERNQA06EM07M zP*Mv_+R!G(%aSg8;0LU1?N4H|hQH(qz>Oq3;?yx1`1pOtvm6EF!EoaE0V8gpjL;z) zgyxx^2E{~_)^9)4#?V%Am>HL%z}eD5wnR$wDmheAS-@_y?*i>fmUl(76Vm6c*Zvko zn~eH!fubP)m9WVW6w`0G%Ium9ctuvn=n9oeaHFs`ZS03UXe;{8QU;Z@& z^`Hy!mHS}5Sl425DLryQi@aC^Z6a!i&ZzAD34hP7kE6%@Ct^jBp8vF^;LZwO5TZ9b zr<<&z52hl-P0qc7m!5hG=!|Jz_A>klJoL9!fm zU4;llzN73k8})ade2Hmt+zE`^LKp0oEBYAKA!tbqnBvSMi05UpSW>aFd90vj_!*s| zct*@=I{zP@^h?}qIw_Z$mti-Q#G%$?!M6VIXonLP!ZR0=NvvKq!#$)n>20#7WGEx zv?qpYlhoocV+@u14RYZHIZC0dG<5um+AvpfGIlj!mk-8>q0e)6@^GRp4dm3LOH~MW zJuw_T@15%85|T_VU||B`hu|oj`@kzAc$`kax`$c1AlYNLM>_bG-}ZCIbt)ENkM{3% zN+r9^Z6HUKx_L>of@H!7Z5lnw4`?D6=7_|y^m3|xH!2nN3qpY_`7%;FlR;Q7CgN2v zz^TB4X=l#4`&j}(0-{}EBV72mh>`pSL?Bn-L5-A^9>dF#b;gqaVT3F%rhk-B2;_Nc zZXPyDnWCTwiHD?VMrsnjitIfcSJi}6dlgMIi1(;#pjR1gvxyoi;AA6E@v7d&Qr{K} zGe09Kx9E?dxH2}abcQ`lEvR%LC!L5ZIr#I;dJG8dsy+8$;USkK z$f!zaqk?zLLv@PGWkVzR$HoB+W_#5pI!_0$YMIE7QTuNC# zM(d~PK;I{L%l08CT{pZ0Mk}u@Rt^(RNA zkZ$fb7MU1q^v9=dz=po;LsM6}^i#JZw&YbOG3sHDc!A?+a##(^Zf*h_w0?}kOx6T}?o9+ZN_&piTjIe3kMJF=23Lcsm`dE?SdyVI^ zn7H>>lm0)|-nYMB%@JICfu${} zKzuacue<^xV2@T%52Ce%aNCC~(g||TrJJ%h$9gURLJVfUQmm8v>jrdRgH}cgO`Dm~ z`Y4hb<$j>OR?C7Pb}J(mvFkB3e8Kh}J9#b#)1o+iFG^Fs!}oCiFAI6se=OwEGnVjM z1C0^7c*w?wqPG=j4(KWtuN-D9Hf?2UF^n#yDzCgpIp=0AElQH8PVi788i>?#ng4N- z-|F>#yU6c{U21V)T&Xdy=w6#;;Vdd#qL9nOJ+Z@+Nd|3gR@7^z$gaKBszm&TVWRqc zjEbxh{jgcsXkjbWR_NlXq9(v#;+YMk3Ix7suQHeFawJP2c5{4qs^-2uFXVUoR`JDO}>DlRVvxr|Y}jb%Mf z8u(Is_t6sXHk?0K>u31G3oGC~n=M;h`&;@W)YY_&aGg#HR!R^8 z*psKkpbI?-EDs21<>Q#bXtzXzv7}*iBA9`cW@YG)mN6($39(ySR*FfE?p^;%)CJ4`NXf=%A zoNvV^6KkWB=XrjC*+_R?o{Tn2ZglzP$5sXiCQrDlQ=K^?Ix#%4J&<&A_%4T0Cv0Atv&*; z5c2p|fS&P)bA>H(Q+GKp&Q;yuSB1{__N-00u2oIZ#(&#A3#&d`LTjl0 z-l*eoZAc%GLSb1@pzFu&3s)}i#*8t=hVk6rZ5Z+{llA~IUWgA?c4_ex*c#=%_PtN@ zd?p+fkdVS6!!$^tUQ8cS8oJk>TLwdZ#S*81%gQhL(Q-O1DpSi0!UUPG6dR!-gGcg?4iObFU1)ZT8_xo3Bc@;NoloUGIts)E;*^JnEm(->1z_qFOcNz=)^M!~*wM-2Q3d=Y%&RQ~mQxerbyvi@ zIfH1?7kT|hKc3kv3SVJxKxTFKR%_r-B$~&Pg6Jr!i%qRsiP^+@KDCONN;7&|wChl% zjPCkWJP}v~o98s9EjI5>sJvRN^grErd8@7(74E_ReB*Q)v88w9gPL({Ag#@*Ki8&> zPA*29Q#&%>9#WxyAo5ME664ja7^kK|til)O!d0(3{HSB^SN0hQp7i3EzWC&y@Vt92 zOYiBYZ!ppe@NDh)w^2aro$={aFQ(J&izdVxrft@Sq5b&2y)xD^8je#ivcO~TxVKD%;b};zXZ3D`|rHn%t^XPQw+s4>uf!vv}9fRV&lH7=37xTgf@ZbtUqv( zaY;fffUlLSLDW6V_@4TrjF1tTVAOfTN||VvDn|PdQ*1X2qGeCblbS&2mS4=j;34IB ztEEcA%mR)P%s@T7PtbSK@wyo&5RmHqK`zTfaBdv7?%fU_Mv}%qY%XAX)zIorj*46@YH{oB&qCM7d z7A^EX-Tr;AoJ6ew|9YmV6r~id;Wa8^fd%e)%WD?)|RE0d4(yoz1@Yo+Z1xQa!ovgh-uut zJ-=X0bewi(gkFDsGDGdg8grtSN-rvyS?FYZEV9IAv|yde(2@mUfNU5OVS{YqbwF3z zMrF@forKg*GNVJ>p5NoHFJ+0*3b0WW!yQX$3Lh!ivXzaBoF6&G3u`$ekck>NU2TB2 zu)d|FR$MvO_;sy`U|Tt{{0%{V7`pf!6l!8U4__&L%~rdA?@yDK-`kRcRp zXES>v5ayOnopndx_xnQNcYgp9E9la{xf^}lG;VKimjK-X?k}f6-|*S(kKSXvP^~e+ zti0#*N1_3MPTU&iE7HZyHwL-CH&x@qfacWo%KI)WtgD4kUt*{d0--m^Ujt6{A{Ilp zE~mHj?|5xoOtA>XG{X%FPkGS4BD)68P$xR0kd$)6?<3yeHbqq(84@sSY~(u}lUf7a zCi%s~Y4J@#cDtnn@=cvojsd7YYSHIg`BS??CA%Sbvs}Q>jWQ{w7c^osFLK=wgq*j7 zR!xq>8~cXXMpvz$A8WY46sgx(n|T82GPjyx{-H#?SsE8^0k3O0AIa(4l~1e#^##j^E?D zKll0zKc12D-n)TirGKmiz$}Z)8zY0zuU>^R8_|e;-sSiUQKaen?J$Z=J`mRAB}T^N zzf_j5N%I64@Mk6OEfPe439&*&ztmEc{f9$tDn(5b*E0~hT3)OQj;rX}tiXzv!Pf)Y zngwu`N(w1D#LrI29Q78)G|0nG(os32Yb<|ZzY3Dsd7Ccd_hc%Tc-1R**fF1Ai<0Ce z_L#J&?ktjc3{+WLyU*O+<5Jo!*6@&)V%m%fD{jw$yt)IhZGvC8KmY060G^)-9t8Nt%tiE-F(K9?wS-&t zS-WXj_Gg?CJBHXktZLf$bz4_Sz~YfP1igJgU0JQ4*lwXDy&?rV^b!qv=^xrH1~?!S zq}{4BNA||+yY$-TnKO|xsukrs)QZd@mhjTXYdOY%R()#L#PmugyeWm#|I%a=J{+@n zjq-?Nsj`mJ*}_}aYCB+Pt0ARabxxf!BuZ=RM2XStW$AsKK)(^lyb*^*)Y9*T)Y5P{ zBQ}Z09eUjpnTR^A4z>nVY}*GXKlt4bfY`H3cn*Lsfi;4r7hwPGJn(@qZ#NKFyT7{2 zh+Ze}cisCZyUn#1(0g<@`q;?0A1uz~xT}zn?Bfbk$7uG8JZr(a3Sk!~^jGdLYnqDR zO%cKHY{q)TBQ(FhZwzpdcMJYSWK zapq!~EapMKgx1mSTMJoG#JSu*qRLJO0mwe11VEOXLBKNRVpPvd+*^_X#ivz169V{X zkiX4I`K&$jNx48t+KftX8wybV;dH?BEpG<$6?eL@>G1Gn+4}lLEB7l1BIH1V=)3iu zTMBgeu*NSUx&S;4b`P0HJOJL_ZkG{jKPzu*oTmhUJnw_%JWme-yld-#E+D~S<2?2C z6=zrld0EFU<*3!y`2}JH);)vHt*Ft`je+QJ)32bn5-aFRalR_wX{~;#jO7bu5k4zD zZEO)K3zST;3b8EpyCYVjCiQPIxcjbStP<_fp@_4Qb3lry!C~|hLy0~)mcD^wkDqfH ze)@%zINVAof;#S;XfI?&17#RUx12nR7K$z{%IAV&T2p$;K?mKy&no*4yU z6*5+#BNiAew*@CuT{0yhH6n+3_rwuLuOPAoE#fsg0xtYlE;^}!s{ewJrQP6!vxL>u zi4J2jw!kUbb5kMWgi8d&!nky%dT%S;oldHt2b}rbNXKlXAL7YqRYg~_5E0|aqY>ty zi}m~m_|2$(6~cz^%>)SCLVeXi44ZNNK8;(cqfJ-a`xb*I>+CI_fIs^M(Bn3OOnJuAJRo48boheU8E@D>?hfg2*I~83Rp{ z4SSNp(l3N}EGSbNC6Jr}6vJnyydpQqQ&EvJ{bD9$0tbdHk%YS;TMurWG50L=yA1EV z`{Q!) zPIyi#DFQF_ex(f&v(PHuPhlL|gd9tvEQU9ShX|w-kO4oBSJgp|D%)BhENEBX71d=z(~&(Luxp~manN8S;}{q5&!C#w~hucZBbxYACR z`-1cesQtoZ{XXMhd!bz`{Tq)((!qa$4sN^}El)5J)dWl4jtw?-T%MW<$;>U9 z?}#lzj!yIG@s)UPc)};~e43@wSqF)+wEcjC94z8 zbMC*B=)B68R`#W&j+YsAuHiJ9D$>lSe(`GrIm{q?neCM1(=j{tT|>1l>Z_8Ka?dEi zzfGZWtQVH!yg!8f)2yb$cx=zgySUBj{M!ufG`uu4ta?U&s8}ZPN52*PM_2+|R%CKG z!o)(nhPgHz0dpYO0+J9zGBBa#oHW7_RkEFs)#Wk_{s$_LQ}_vYj%t9ZvL=$Qt_7b_ zxMV;!m@Bx|ItJ7xFHWBG*@gEBfb4OsA{mt?6)ObzAvsEh~yq?)wXJ8YMol$V+1^iG$~y_yjl#U zf4hP?1kP(3TN|D9;%H&a|KwhGCyLN6*IFkmU64wNjnYaAR;+RrCX%t5R6I~4%kAfQ zYb{CtgDNal%^E$UbUSR@#mUTWl`1{RD?bGGi_l(rnz(wO3$IdS38&j;H7Z$5k}VH> zyu=_piT%vPoAF^p<{E-vVh#>V>aST>XW=n5ojoxv-{k4=5GyZBsQpOP6*}05`DW?v zHM2R5`+vgG|8;|h1h0fxV-&1j(9daoyTN6Ar8m}JGXK%-2L_i~Qf7F}fU1^sPy>rgN&whh-l}`S=0ZbkRNY@yHcrcmg@5HV$Dl#&#l4G z8R@amGhz5@MPO63bLFgRPOz3!$?8?9`j#3qVEgs3ZwrgyI(uQ_nb$2aqq)n^G{Ea8 z*f%1i-a7lt_yS5mS`w5|FRV?I2W6#+mTxvv%8@B?<)am56zLGwNu`6Q^x0aZ*tiOb z-X`{tD0f4rP|6`}MQnSah(+J&#R=tGvZq^(tCTJU2`_F`Ab(;8(NWQiEwKClWH+n) z0ly^;SRN`0?lGcB^2vxMvNO2KGbQSqC5>9dpyYyYRwxq2t&q@T=iu2QnVt8HOmLu< zW?%T7B~A>e=b%9&_`@|g3g3G8G01>$*nAP1p-u|s^LING&*}oy7X(r&V#N%lffte{ z#*&j;YA|#U-|>g=3A)@VM1!q!r2cBC&P1xEj+ZCf@4s*0?^XU|{+`mW>=CR-Lea$f zjK5n`mn-=PP{Jx=)ROw)JJJd#qRNj>olN2r6t!=R4+FKq5|cBFh}wkiV6XHC$tAW9 z&75wEn`jd@oGGuQHc}+_PpK1)Fxe7j2K?>e>bOK@&L+fBcOsZXqU zh>xISeV5IqWWm60%ax`ERb-r&cp0!%14jr31Ia(DejjRVyJ@dKSnW`w3#gWx$i&9_ zMLw3mq6flB=kQGXB+H$@{(a4doY?+{@}tS#jE72LRa_m6Xwo)48<&K>K1FA~2!+HO z-3dOsWI6)&%FmDppCJjFp6(3goIjJrjSPI5P|hPk<+i!+f?2-hb|Or#*+btC#C$?6 zIk;=qC?20s=grXkk$R1YJZ_C}vj(brh-u=>frhE1&0X=hov_p`_=uch zm6NnVZiGl%&?~{yxi}@5kUmz1eepV&CNsiYehOA8kaNpzy6oo#cx0ei#^eYHcKb^_ z*~Wv~ZzFVy!mZ0W)NH5?v;04^*hVnAsqp%HVPw$Y#;57O4R2E>2XuqGZD9odt>U-R zMv<`k7kQ338K6WQ@cI&%k#LR)8=@N1-A}jteXB;Qwqsbo#2DVYQN;gA=mV2yxGU<2 zHOCu_S0vQ$^tD70>pY@~ey0uLTuXG^j8he+6Y$d~{Fgsccw{bo`xuY}yjQ>vV+cq< zfys67u`QPDb50<%G}9nO5HnlwEC5ygygRu4***FK40{o*K+d?DAtKF(h*^X~XZv>C zP}Qv>I}&AtH^T&P>Q)Zn|Nl*iA&*s>e^5&)f6%@Vz0gTe3y$y1NCE>)a33#?*g3r= zEDQd98^SNBL`jTb{-5!%Do{J|Q;rbKpGH?9BGc8qw=qE=sync;jzM6?==vOZFb)UN z+{3gcBwbN+r)RnvAU7{T7#jL}AGoVGR;=v655(ut&gqZZ!JwScw;RO!^*JTuwr+=E zj@baeS1A0pJVb!Uh1|kNB!=)Dig1j@Z6riJQb~;^{-1*$TbQ74BWUiHd$yIAWL?=K z1d&V3;(&*I0WorOfe4q-q=*qUliJ5YC2N~$JIKs>-S~?0!5vd-1QSl|djHqg*R=z1 zIBIVu3X>;@FkMmltnr6$59TF-?An7-Z>IIZ)tooamUFhc3wh{)G``Ldv$ypKOA4QI zMS@y~m2kYX6nxGEZ7Rur$8^>FJ5?IOoJ%oVl33-gi58?0DjR*Rz)ORal$Qz#4YPObFl@Tq-F& zUM2203=Wdpllvahy}QCi1f}) zt!hTmM0rC%RLuuO8u0GdFz)A?Mr2y4)d$@Ei8^_Vcqilx$~HVWt19l%t6l+>Fkg*u#}U|` zXk}kidx~7IpWqS`93d9eZZSn7P4$d_3=&S*SrG*_%ivbnzQ5ds|E?)iu~%Qm{~$4= zS4CPO#gI`g6}G`|Lw54BCf{Wf5sC8aU!791fl>Plv&stRWx2c#q6oKv|_{=nLMFwMsb^J;|8sfq$Lg13Cd!_w`;@2)ua3-7AKJe zJV#!GT5J_b2MEE~ix%ZZk=2R58P6A(@lz*FrsLRWk(LTKDe7q*@QEGC`lm(KlNvS^ z6nQv2vE2m<=D~j5j2|xvZDZJgY{V#|8K@lf5!n`cfGAhGT2f&Wq6@d;d{0A7I+?=k zB!3QYg*4}@gsB-YRs@eaQ>O*cRJIN>PW7>2sX=s%UHqIP6-SMOE5$yvRi;&3nif+iij(Rp zD(WF#cDUI(0wQrHEd7u#cOU7sB>DFBbCHGivJdI(^siQMd=Su}ZKfeHs_IEx$mGFm z*!j+~w6TN5Muh3ePbO*cvZ@3})TiiwlQq{$R5-)Z*fE1F5QpR*EC!1pJ212<(&CjN zA{CDv{3`tth-4?_v!VOD9bLXNxH-DO2j&*#Kl459guM%I1QqA~poa~cNTHKmK*GGI z=r&lT-Tez+up?gz&Kn!D$_fo8Je;~M%@;M8McJZXs+y#mbSuPYJqBLD^qZzSd_euJ zpT%M&DqyPs?q+1eLeNNMDhp-+72xaOpQ0N=8mUD<@qJ%>>z_<`9T6$Hv@sS2vkrYv zlxf8F@%(U8oiwGY0_Rp0guOdo-fI{f9^_*yxJ9RAdC<8h%`~P3eTFpasH$<0ysIyt z8=oDNMuhvg3C5pb1+jw@x_3%)j5_kS`SmeA`BUr2Uz99v)J~C$%k;-1Zvsyv+@t zcR!a2bonv1TSz8t%fUIg)bxw#voz*SRSOGzvcy0gTc)27Ku-)MB#3RiiTou-m9;%u zHThO9eTZ5rgo~JTmiR^5rqnx#6=kK9a>>1eN~8Ac43qJe5y$>-gD-^@-R74}9=Nd! zmC|FU*R8_7Z5QxcqQ~HcztVuIW%4$>OfzZdOSM|oU);+#DRh^#f4CQ}&$#v?PPVBf zq%Y#7Lp67RYCq4CAzIghx%ri>)4n+L7Jl|U(WWc?pGbH_d#$<;2>n1N4%|se$qdACHB->-?u@;{4cXCC}F^Pu5A*GSt=OFQUb3p~oz5duy?C$Yk=o4$;5b zgLBO|Zn(b9gVCGPkC;911j$z9=!cCYK0PFVP4*2!f%8K;P{Agn4=sCco|n0#YA>W$+bnp|M@Bu2Fhno<4{Ib zsxzo0Inq*|Q{`Dhsc6WJKyK2M)x)|YU3U&sLeL4X@2Q&_K)FU@;Vx8V_R+zQQ=oBrTzLoN~MheMcP=?$-}&n<}G2C624vkv69o`$qxSN zdU-mqt+>$UJ%2J*d8uB#>;0$dN1XTSEuXqg5V}iENLuGrHm9@av$8$COENmkBqK)E z%aApj<9JGOt#ijo^T9erpPK|G+xP1q403EqJxfsxtww_4{V_yu3oVUWmlav zIVg4)=cs9~v_~b_i9~@m&Y;tniNkU^K=p^tN@adP30}ZEovE%ES_ie0OC*IY%DDXW z6>nqBxVg7VYI=g2F6ccnPrpNctqUJc{NY<3>KS$;jDDUmjw3VoJLA}pFF=0-F_YZ1 zIcM*!@0pKAoj0@Lo6G<>MhV_qMj2IVuIz4va|d9y!jkqn(6b-rIhFp@pDr=8 zp^*4SyC*an+B3%$(c!}}DM-QLPNkeVdYw8mEFHaiS|eCk{9U>lj)h_0Ahn1OuTB5Ukdo62JM5Z{>b?>V@*gaqo~fNGZP${ zhQ}WOGqR%i`JgGimR=GA*0wRgb3nV`lj4a(qh|6LP#tZG1>){XgFr`&sgrS#A=*qG z--5b_n~$@Cor}{)#XCNAoERA1s0{%~VZx1{R=T?~VBWMc5j6YsnZ1zZH8Z=c$#N7t z9m#|nKi6bH>^rDgZ`LW^G({I2Es|ajMsis%V(13 ze!L7tqv#j|r7kc?O+CAG%%H6@P;T8|_h~;$i6(TeKkXMKxC^ z2su89-%YKhT(QLtZ-V@O&az-=I-2*ird%-#VF^<0$S~6dI{~)*!UI6*WPA`7@ zvQDk|aC=@wwr0^h)~gdSgiNadu+%Xl%ev$ z4Y-;cE)_kum&byFm8&A!8z{{7D|OluLR3;f!KWjunqCDY_&fMtlnZ19Rz+c))0 zeQG{~0fmqi&6{V1naEzT;YWCvc=!vLsm|r*RSaVZCS={UtR(v=uC)6p{nU}zkgX!l zCBR_wT*2;5i)ah4=n@iJ%FZiHA7sq#20PYZ?G9=@gPv&iG>nHrzg9QVf}S!H<@Dc% z@^vkoAukr3RlKm}%*BrZx2fxY$6>}ZoVzz{k;)E|-9PTfSH{m`fp-t1Utj+pi$krl zZiSD&EM2(&ISFHN9A0VKV^|WTHCd=vW|x6kW8DX#N>}+qDVw#Qf%DmD>bo9Y7KC{ji_D~34et!5}X{xs>nD>lf z+(Z0ZwC@`>nNsz0r%@yLc9)dNO@y-!#uGvrWyZ4TZ*|v0niTC!G5q#Co>K^*pBmck z=~;66QGmV=wYi10cdUk_n_y(ZyV*<|rkcPS7>ya|50W3EfJmPiQ(Oo6;noe(*+{4n zA_Yf*<31_eT%&7mLq?Uq=(~Z-_K9_a(uw2r&-5L;@%)+L@h&{>dM_2pO3T0 z-%pd2T8UK9?w86W(umqmMl0{u2&U(O?w`VO3~?khBD28aWoT z0LRR${4Y`P#1rXX`iGOxp{2g=pF2hCM2$`sPeiS)SLhInm8@i)ot=)Gt^qZb8^lNO zP2cz4xVg8c>8Jlh=3U$DMB8o*xLLX)pN7KL?_8UmH6uMtKGsF3KRKGBb7zb^hz^VN zJ8S)Ht^8HJN~>qZ4%CG+iVwAbGjrbq6m(EpG1sQTH&IM#5_A*}I{%V0#c2dx&x`mG zM`Mm0Ez`c4hJjufJ9o zx>A6xn!IJHJA4eHMla~-$&z+{(FRNr2y?KUc;qjtp{KpT@%s3{^}0R<{@mf<|92L~ zo<w3T1NZnyQdA_Zc7Az%hb5m&?JowUZ)qW${(LR;%ZRd zTEq_1fZ)Fh-&fCfUf$iZk?CLyR;ENNN2`pv7FRz>R;pN=J-h@PcTwiSBa>qJ(H2(% zTrz07Wfj#-=Fm%QZV>8O9IRJJE{#I?rC6Nxj?CjB%y~#|Vi}dcdf?e*(48F)G^g4g z>)RB&@8iA+;Nvqo0ed8`wr+D?K>J%OTcE;U&#>9g9p9zJx#nE|riCSq;r@s}A4Jm{ zFvvZh6uAn}eDrey9-WBEYw(XT1HLXx`gVBlGW zgn9gO3PM-Qi%r3C1w*?TWWgffQebN%H=1g39#yB%=`o>=-u#FLb;xll9G85RH73d{ z52cm2!2(fFhDxbhol1v2{Q<5ZUQS|{NegguhRkEI(%j5_>h=nc)@r_rpUBdM_N!g0 zCE^iy$9kqKGuxMw-Xd3fT#QvnA?BoSBzT|s6o%XzX{wnW5CrJ!jKZrIDf7X*3s?sH zq({!}ZAvo$ol9X+P7dOfkY)4nfM}Kn(UV5E&v`VmP*k@Tr-<(anVpAw}l84JXm>ebF z&Cp>#5p^tF_6VAcQbl2dP(?xWH>*h^>+03H%8J)oN3i9)Wce8ZKjLjWA#m2MjCQCN8`W#%Ktu%xaCbV7aGvh!c>>iq zfPa&Q)#iUDoLDt;Vtn<$O{YI{2S2~gNgsenx$q7`kpai*>j5DL(hz$83>6r@RX(P38HWmSf(Fu zzRzA&G}#n$N`4~D5W%l$H!rm_d^zamGhfTIV8%*gyD>B3SjSs;3pbgoje7)mYrGL|Ox#T_%BtN*P3t>;se7SIsY*}L$ z621WF+V(?J_cufgv>3wRs6X}YIe!Q5YTe^Ps({Og&nUU*Gr+TpK{fuaKeYoYr)j=_ zUH^p6x@4UC);+j=O;e_YDqEZO2r4r2&b5Kl78 zNUG5u>hBqi0g23&tHmK6OC3$jAqjJ2tdYu*KdKKVzY1s7A{@FmUrTwiC6+CX$kGyancxGb-1Q3tREV2f&Hg&SM;(U5@ zChPrv%Fv_0pVf`@-cvM_wD>&L)1Tw+}gW`&!Vs9zp z=k)82|*eU$=}44lP@uX{wabBjp4GiL(5z7(7_S}OZy`reg!Ul_tkEWW;Bq4PCYKg zY?oys_Xc2Ls8)-XTjyBcv5A>`^`-n87P~+NaT=Ny zm(vT(tCG#plF6!Ydtsjt!`SYF?Q>+y@~1|TI*lUw%F=c}6B9=6qQIC6!;!EaNjw8$ z3P2~_My085ZHM$xqQ$8WdAg0D$xaMRN)W&q7y_wDToq5glHem>NpLq*Y2vl7B)AsJ zo=C#stBWD@W)Qmn?gMxUYfaJ1&^R~^6vM=x;wJpJ2I~akDZvc8GEqL`jLk5^@LDlj zzfNkSzP4uc$smP;p6LaVIKsLR_a^NkGv^)As}tqkzMkxDtTgK;PjQ*V?R_iO z!bs4f=JO|`9AV7bvOuFv%vSzIG;xXLIbn%XyC=bpmVKuPC8%(sol^8y^#tYJ=CPV- z(KVTjlxV}<<@NwCb*aL=#&3b&k6cvOYZ;`Q{ItfCUg5K_ii;IIj}^W~k!)xTy+S4f7WSBTzK1wsnEJ4U7fVz92s=dtri~I6*Pu zviDz6@X6@~Q9n|Nax*Ao$0wP|aXL=m$IpW*@GN%|%aEjuXwV|#{7FdGKuh9mWiGRy z>LY>)H%P}5Qx8g|IxK?uekH#;{@E`-Bs3(1CtR=q>@)YfSrda1uxMX z`AJ5fI6+HRKQ`PhwICnPaql|F`VF*EpU57q&!$k~JteF6%jsCqj*t$kQ1x?kXvGL` zSqPTJH1h*>NW}*LV|*+8%ZmYETB`x7zXn(gQW=|zA{-H4>Eh)-i0Z+#k1Oi zz~rx3U?IPhfEJ-zi;8BTDG>!$`{N2P!B$T!;Ite~QM+D8Kgj{zZgn=hSEF3}+r2jo z^KIY&6GL>B-nw&+e)^3(7bDqB6S9~n2B2`y*N#Ys?H zWy?Th!y8p5GVJo>0>ThFWtA4%Nug!3NBv}8?a@^*mI@)$J`yZ+HS8*_NbRalUL#sM zZ|Yl8Q{x@mmDUhAwA`XR^IaCW*<%JZiYjwX$EF}+TLBBPCe|NtNeW|yfP~odna}63 z5b(Hf^3_FvdK4?aj*JFYZA_i-fnCDPClqqsQv18kNGH7zD(bLEzPyiHp5R}%%IGgH zb%M-6kHu$21`4rrzezmJ_@G6h-7%pXehz5*qnxHp=)mkq{U8Ov)6Y2=3+7h46o(IQ znJ-ro^bQw$T?CnEPQalF1StfKBt97khEnEtF{WPA_C%A*hD8{1aeG$kxhCpQ1WbY( z(J#*BziGI~6^ZDDbA!l$3Hz0)jjQd+Mn!2Y>L2M}k(@SLE&iGlhsrj`U?N@~Wie-h z#X4pEviG~k4JrQKhjyn^+9@{>{ z75&OhS9L#@7i*z@C0jS4CO*3QPkMmLfLM_hQ>?6xQH7=_ z303Si(q+`ENzI1hn^{EX6A`w(wTZ2LHH_ zU#?i9 zBwO$o`g zMGk7VJK>_3QxjbUA(XPo$jL6M{fnlxK!RNK58w!kLnadyedDuFh)0(KSzB{~eCcaN z>QWM_LNMc&q;${Hf7Lk4YCjpV)8ek{Q1|uiO)H zI7PyG!4@4vk8s1Jf2o4o-{eg>3d&wVP0GmD*ozxwMhQ3hJ(4_}NYX$HYU8C^m25++ zv7$a_rJ)yq*|%L~DB!O^qk&H{r$&P?JKPaVHJw&Y4FF_C+t>3cIS`vB1%z{122k2) zvAYHkrI4V8=9s4Y_GnUr+o3!+QG70Id2KW>#LT`)Jj5OMkfDsazwu3o+QkL;Q4DDw zWm@+?X%TB~e;wA-MGWrd^WX`*pz!tggdZ^GxkGV_2L>E}l*{3rM%4ojnqf|Kg{Lgp z)Dc@j@n4WQzY!y$^I|*11I3_xd_SgQfuW^;KS{$@G?I?&XH4hca|63i-h2D)>YlNH zKqNwOHj>6|&LFU+JbRD)f{}tIe(*nBB=lsc^*{XWGA)zW3?DjEjeg%;zJIqB=ph#d zuFHQoE^e?LlY+WGMd^`CMC5|=gY_PZgHx?|_kvXVrlCGdZU=~~y6wu^pD=suk^CmK`k8F(1gZuuN=QLcOqZxluyObC zgGOXeJ3yUA37s4c$!B`)uXdtGPI7_!J9e$+0`_=3c4>X!kf>Kay1^qM_LqcgCCGJ5n?=9qTBxFzA4!iahXT<#l3z$K3Y_hP2 zoZ%57OA!diV04JVJ%ep943(O@9&+1pO_3G+zh&x zZ~^*4)Wx$9kFUCQzKjO5p`R8t(XGNaS9Y`Zn)B&cF*NR0$di}V*t5!i#LGoh&nln= zXULhH0FCM~VHZCdUD)5aa1m?0IdEr2)V>bOqdk8UeS01}CuTejJSSm1{KCP#j;xU6 zs@NG4!7$ZXxkWE1*}G4#J1)2}qK zVu`BJRaf~zTpH4z9caoR?Xwa=8LU8N)@8l)!!e3lu2TuoW;iMg<(N73aRet&+mfk< z@eejuohU4Q$Wu_~z`rVfbcSIDHd!(jX|8>bkZej;|5ftCLn(gLjF*r$H4j(XY=iad zhNRT6X4gf?1@6o816EfZswm#$OhB0*8CVOI$k?6Z0W;5MR$qnN`!efv#;6V(7`xs^=C!K}s>iCc zYGfm{JMW3kfwUxhts)^afkNofYz9c%6lqj64e2d{?p(T~JL%||4hk1wwdAwu}m z_swOeD8V{wwSk)nD=Gx3#DC2r(dXdu)jQSs+Fr_V+ZJ&8vho=$dY36%usirguT_9H zm{bXL#uCH&vZ|^bgh9sdtNPm5ze#E_nPQ1#1emi-qIwnT#TB>_9lL!h;XcM4M;g&c zrNET$`=M5q2HX`KC+6rrUC}9x{hlDh1RT(ifg|?fX6)vw!ha7q{>4G_Ng59Z$k3t( zBdwGO5vUhpd&MzKzzekh9PyWMsMuxYg0AiMK*Kcqi?AR^_n4=virT8d8#AnL8%3RW zo6$>vL!xHAwg+x&lSCchA!Q<%huF&@5_(uhT@L}jQn_es)Wn^i&=w6(^Lsc<8}TC_ zIF)$iQn9OIfPLR@?c@T2ESND3Mz}ZnVA>NAG%_s25HrF#ZMxHT)X4OLMFzE(gUzLB zpl~jdg!aocXoa($2O%}V+emaN(?FE#Yl14xQhQOeFvwIrCi=Cbja0#l#6MjLpGfdq z(=PpARWZqvAT|F{6$9V3H~n9!Vu%1*EgTZU+GClIAv%p-@9be7!r2&)JUF_GWJqx6 zdM+%lj0lc7YXRAs3LXjru>DQMm|-J%wM`T-hG&qoWg09n4w1}VxR#aR@hVJ?04TY# zk8yBK59}CP8}~H0AX0mvfh0wLKvWWAvVr=TQ~L{Z4z)|F80O*tbGG`=W4lJ2ubi6S z2`RH3-rO!m*%hD|rQ)WVRj;hwjCmeUD;Zg6`|}Hj?ISj#J|WN4Si=;>56^6st}vo6 z3(DvjA?I9!p{)c&7#*vho8k@*Scv!XKm>f`@Jqe;R(8!2meS6s&wP?-4WoDU0XsW; z7pH(7jqG0w^^Zik@X4nm{#^KqAirz!OsPRt9~BeJrz!t;jW61noOZCY6b{x;Nj!F7tZ71%z0{cogFmY+g7UeW9SyOn zt!%8;R{prjwqK@NHSguC8s^*-DuBE! z7>OqY^~PW3;iWP#8{7{FD0Ce?j)j*)q`V-wnJT;z=?u7>t%wQhGDYVVFx(bO=(biriO%*7HH6F#H1to7$vt-O&d zIZ%%jBiJ*}!F+#k4*aI~8R1XH-`;N1!O=Hc#Cs5rrIv8c9#r3$X0?<*xespNX(^Yr z&fSNNvnQ#b#fGRDFSP3eLYneOLy^2*x|+sgk(zb{Qf-WRS47ja5=6o|wvE&tLxZFW zU+HH>CA4DaXy2rAQn0^#W0~qa<`1bJZJ{rG%K0_vRr6rcH5z;;q);WFbs7$;7tl3` zUN?;a%Z=2qb7b23`|W}4V5&Jq;Smn6XOGqDrIq(p9U}*PZJ1SdzGn^uYs~G`+&K3$ z$N!Z9Cc7a#O=Er|ONAq-X?AIX@d1UvMdExAqD5bmYZ_zm;~!a%$AQOZ$ib*7bxnn#HXn&*A|1q4o7m05z-He|EntzATUcccp+@%#?ElAN+v^ zEWp+4#?NiB&l_SsJuzUR#z?T$_Izb_lscSWYSA6eZj;+M;qoNJ169=>O0(PNDnlPq z!7TQBQn7pgJ+i%VSzM2?no#!ZTff7xItq27rV+j_!?}WgAY3<8iS13tk~f~>z7m4D zRnp5AcKqyc&Nlthds0gY<>~t{KM@JNAf26-Zlkvs0f90=FK2 zO>-CdbiHNYif*J+pJt?nE`~Nslm~s2$sl;&2AI?xG-?cU(~3uLtb^_cFQCM>F49U} zO@F7lrn`koDkA)L;N(j{xzwLz3em~x!+Jz9ov(^r6O8fc)5_Lx#Z(YdR{uDo_KnZ( zW2L7b%jYR4%Y|~{Re*QC>W9#12d-ZI(FV`? z82B+K`wL`O!3&>U?Gza%*n^aWlI4Zxl;QxaX400J<4AWK=m3SfAZi_QJ(Hrwqa)vD z-2V$1%x546Raf^z#7#tuF4i$bn0yY*^O8`sD~FBBr&nS1>q+ZoZoa2#!Ll8rYwc?E zmqe&01q-=o9nNlS6(G2Tic5<@j&p@Mjz+Mrln4SoLf8sHUZ&v_rJ(=dhZhe8<)6;ph3Bym+RV zhDa|~s)TIswytGN(=8@m-`>vN>_K(XSwT2j=YhVB@ujF{0HO z$-%E3_T(ZaRU4e9R}!i+RFTVFxi-(5!w#T;so4Z{=%8D6o4o$&e0}~!6v?A(C@}e5 z4SAW@Akn?sDvwQL<$ewyzXsKlYcUOd3__*JUt1#5%xYqLq^S(cI%>z>Ec0)D6IGNC zU8G+NL~H@5v9-_ee$4V*ONI8;elm|-m(33|juwpRx{4#+2LL@{n*(&lW@f*#-sop5 zm{5|IQVEdQ+1Ws2b?@+Wch8X>$Z$V&T;Pj20_5A=-nJeABDjRa{@o3I*_sAg?ii#6 z!!^CS*UA!-e6&7=dB-Ex|CRk4OZZ{wJ`8LQ&|U4p>~$ZrggtN;>C&9L<%V)=7HGaU z>j&R_^@Pd!cw1BFa$5Mfr>{@sGLt)}h%}^HMNw5xhgm^UU1Qqv9)QsTz3UCP@2X@rcy6$P zO*D6v8cAg-$xj?&nil-NE3y1bLe+91tor33D(MeXT6n_)4*;QXh z_r|Vb&o>ZROj|X1Olg*|cyBZOnwC_m9K~dPr$g4M6MuaXX0oO;D{Wu6Aagc`&@njw zbNwatTs5H^kTrQ}WL0|pnS)^-H$r;<7fLvoQEe_fqpp&s)nTSMb=&&BJM)85BlfaM z9O(<`fk(`^;s1kxfIx8E`}Ia|KE-(9b>6YXX=Jih@T$ya5f`$Gk@h!j@3E8R9iUwr ziSQ3?`&_t~MP2T5D_kgFKiu!!L=(g!@!n#c;7E?kDw05H>Am#GNfM`KtoE9R)lk_? zt@~AQiK;2a5QeHh*Tb0cdFRMY?bhJ7hV^Vho>@!`qnK%cfh*LU^D#d|>a*5Z=k8qdi z=I@W1x)Dqk4kD1E7-YQ~u~>JVZY9gq;$@&+lViwILFc2$4C>X2M<{o%Qjypc->f-> zRV309kCYEfP2(mP21XB0D}7ZHfPQof0qzYgotQ`aD5`zn_q%p=y9q#u&e+#nf}=)a zhVsY&t6^GVE&B}J6m0v`c1WEA93M6{ZNi$(i+GT+Fx)&|K7X!^<~N*|?_|8ggxmDv zwR)+q+N}Gyf#T$ysxpVSN9sHDTV~iYVNg`0+#~$i_WLpoTiaqU z29W|Psw;FfEHn(J1aef8Cj-GChD0ir$-hgMn=J+gPOtSmnR9S0h!GzxAuE28^LGtUY9 z7`3^gSTu%1a@2sTsdj0J&%h}1KZh@nmBobcl%$MYTs|)^c%4gQ?lo}>gaW9fhSn^H`)kK49pjE6Zt+ArB;pkHm%ECkQ~1rBD+1{@5l5Lj1v0;U>um-Qi9PrFrcF# zSz`l#F1bd-ZXTs3A4Cj?R-PqaR?yYTuT6%Og$It0(I1I~iN#;Hih%TwWyu(e4uKtO z3{B^(OocY>`7w&f!@l}zYh5RZbV6&l_pv%ng|dR6vN#01b7kTPGmq0k0qBczs0IZ6 z)4pGSv)IAlSU&}jK+S5j#W*e}wk-|;v<<|%-lJBS9H@GFTsy1jBA2e`R>&>x!S^8u z7$#{L|I~MMjE+0gLx}TDX^ch)F^Zuh1q2i$2jc3vZ-Q?PDlzR z(q{V$KvLo|s)hz^!ldgX8QRh%~X)u~F)cy`I*7 zO%P-2o+}DP99%GaDZ#Xa3wSZXRu)F`jdfMgWS>aJvzMQTH9Fh^`F$fu+p&(SjFyxK z+DFS|;jdjaQg_1{QKj}}suD!%G*%d4KFLY-9MVym;xQzc@AT6o!v|wGDHh+bOauiI zID*zilmB2k{C#=pKlI=?q7|9Pv_`$S<){cUacL;$BW_AB#!vp4YpB;N>B@BrQRkR_ zXmI;v*~cK9e9h+ftwII}W)9flW^ekvrB64CY%X1^c9D(WsPg#5)qqNrKxc=S50eKp z8j}P^K4~&80mK(3ixWVekK1`L{kM5mI}qmmPzvy9vxtHVmLAYF(nOI+1uLz2LyI<3 z>+{?JL1=15m{7%+;8{;Jo3*9@%&?Cn5Jay+w<;J?8aodrm4K2pm(XZ@^5jGrUO3v= z3>?v%EYbI0!{He%G=5~9&6-sl*B2gvNR=y5kqd8B*~pQ~4~mFG=;TgpFqNT}nO^mo zaC*a;`1K8Zzxs$Vkk6S{X$Q1wb?Pi7m_6x_smo0dS6OYDFs0mbekO!r(ux$XD$*GY zAC@=bANhgnz^n``Fvv4kPrw9up}E%ZiCGtZ!}}JItPHLi04*=3l=E}7xyE1M8Y!w> zR>@W3!xJC&VYVKiiHhXgyWj4BIAJ^k_jYCW-4tqHs?l7gkf3nk=gOQDN)pOFEy^SY z<_ZRly>Z1>^li7kRKdXBfaVXJ$72_cD81{j$Ie zA2BqUrumupqTa}Qr|)JkMwY~EOueGAoJ~bwidU;O%F%TV2BTm{aKj5>GKb2ymnC+% zYD$ba`#PLfpw6wtg(4QBn<$&Lj7SAun8ZljS}hn&G3s>d{#aGMQ`p*V>P8agft1X&bHoR^lkA#3kHd&2B%(wohk1*+k#1DUGG=9pIy?Z?w--`Vlwe zaqJ&WLcr+K9I`SQT-RS>5mOeRs$CQjAt3auze2Qr?VVS=n3c!0dI z9m*SHsHA+B+_^9(a-Jz%F1Mf-P8NneZLB^5px_cH@w@>c|dAqEwZZc zH$SBTWHzD=bkA>#_KK0wFE`+qEdk}2f_HD80yMyv7CxiPw3)6~d%Jx8l(SW3!e=bZ z5Lal_Gvcz42|eJu zJg&Pw#3;^U)sB35EunhQ=AD7Iy|q3sRItT5%UG)mHPD>67x^?P##7Wvi? zj1>)WE= zvDaV2ir-EuCrAw`3v1T%DYOZG(|nKi?uU}vuQS>UCRFJ|f|`S#7Tl-l46cVlL?HjG zuD9EE?2b|hN=8GX3g!^+wzJ)8&leI+i}j;_g=E``xrQHV<$Mxc%SX`VMeHb4m{Cs% z^D@>RCAcX}AHrO{Az&9i>@=IngjP42>6vY<3r8Q41KrIZG3C2HsKH9&FebP!*x~c0 zY49g-mHf2q(q95B$m>Z{H_eg~GCVc-L8+?@X?o`)uIVu)vcPG&%; z@R1t_$LgDiBsjE37FHDHekz3-YZFVPj zb^Y?YuamD+EdkKk&DQAx=twH3rxm_WJBX~e3%T)tXtVd%{rwu34!n>&4M$-Q z8y2-+T4#Y|B=Wt40*;ofncscSy~G`=uIvcB-$|xn!GTgCB}{WJKP&GfzbP2Y5con1 z%b-pi_wZ)j^(5)()nxoDN)+|aZGRx-vnV0~KLI|kj~6T4Aa!vP0peNV`+i`k(jPNc zM(ZcNn2*%L_ng(b84*Wq3BySdQQK+vngTINriGwN3NY$aynJ@%h~ubtS1cy7zG^==~zZ7#Ot;-Y9 zI=vh}F=vl4MwGX2C<$HZ zYBmUi)xtKnE)$hioT`K^3YkOCOTqiu8Uj8+8d;gM#vx`YZ)}NX`Y9&$iIyVzWYOxunF$;^Il)v?l zjv^9#AHq~pEM6cYO*zudR%oE??Qz#o>aQp#m$6iqp`*oB7EE)H6<#rLwOCq2VyvsB zXP}ItZGeiFJ~SU(A%ISsuaIO=-=vvLgCIQsKP2ZLQUNlZNOyP*lS@>R`CC!9#gd6F z=~=!rPDjKxh52bQ!dQxd{5?=sUS0Vjpj1OWk(DuN;B+E?W-gg0Y9Y#kS?gQAbmByY zVKDx#-ZC*dKXa7CWA?tMHJlmFau^8~%3E8xK80BWu8+{xgp_nr#X!1yQ)?1w_|p+j zb*xvxeprlDu>R^*Iq38*2Z&R~eP>ikxiT`pDsUFgTpCxbs(4B%-zgd8E?8vljuA}2 zUsjT4naOgtLRv8CF9HkUb00JtGB(x6rU@a4B0E^p?)?oQ8j-1Fr6or-)p(PT3Q(2m z<X;p8&|>o5s&Igc?sZ*hT< z4=$>Ll1cb(>8`a7OQhvJi|uenh1+oj)dy3^YRk4sSC09DZ#29>rX4&ugcW73MGIi5 zWS?fV8f5dMf!8^@BiTTfLWRI~VcFQF(kg1h4qcz#1;G181l;imPonrIJ7Mj2NUSkb z$e4E&`{Hs_!&dYAfhUw=+_AR^3w2BI;@*FK3io+j{IxdWWFZB7#{oJtZ8T-FOpfs)M|RAV1o|^~+2G^5Tb%wc z_THhf5~z#Pj&0j^QZXyGZB(3!E4FRhc5ZCDV!LA7x}Ep?20iLoPrCd431@Q7*?T=} zt(!p3m3a)|9!fsa+0JL8xg+jM=_%o$*Rwx(xIQ^X+;%ntGQ9Xnlg2D?u|C>YW8Rt( zW@DWOHBQJ-WK(w~`QRVJXQ|i?@oicM)VIa?!}ED+-aG+(P!V8yGtvBv^)!xT3bL#} zM9#8XxWiVqAphTIoBgL}rB=yxuhfU&|jdjc*kx;>5 zpvRm9jRlh#pr|5iriOTAT9v%nWn85M5&BC8v^@ecOe0sIr$(1AuHTOiWN@O|5I8X!^kSk zBDoqDvPYbDyQYUn!gR!30rwwlf>2IDX&R}iVHnIZ$SehItyt-67gs7&l)4lR6FJWrzbealXUQsX#3ZkU%wJu$-kFR} z{dv#6IFLnNU-g=kT}EaI@LyPIXF7^~8TE-!waX)`A0Rq(4TBRRmw?(qILd8UCjgWnGUo~2PEl%k^9vm(i zy5C?E-Ap=sq?NB^rcX$-c`DJ*OF=?GPPA;Jy0I?NAaA3)*0S)4YJZ+AeB2tg)rD=# z{c#S+0721^;r)Cy+Bb`nYS#`D2L1FDfPe$Bpqy6O_7}y_o-Y^E2@6h%%d0pB-&Ft1o&yCvqRL_V0aDm8(w@X=4+U zPbd9yT^M5U(O(6wZTs?e5={8Tv(?v9nM^cz@KAdyMNx`skcLATD#& zV1Ip>x)j-<{wd9?qa+mn5~&9RJ~qI!B7M;^o^Op0C9k+<&4_2eLHkr|cB;8tGh-%TD*a(zJaLmJwi~{s3of+T-@3Rg zx~LYuJg_wvs(rBF&5uxZR^e-7<6U*8-<*`KUN+zZhy}qzf368 zAF-;7Q5jEH$sq{yxHMi-1K07(=NYT1bztv9iXG`(x!S4EMEgo^&**1#WOVap`M-1% z)Om=L+`Q8Ht**Omu1L2fCl)!6De0-a5k{J>pXjEuGaViSz65nG1wN-3*QouA6axI1xIT|l; z`gkTxO*isJ)bF~;O*48Z_!P=N-dI8LNi_@axtm zE}nMu$5CbU4%$F1$!Z%?fi;j^=Pd%Ks(<5Np5N$cE z8!buOlGZUztZ<9M|Pr2jnM10qNcW`4L`Z{OZ%e04?~zmNH`&$RE>@L6*Np~8c> zOX+OCw2a95s^ua+X|#=qHXSYl-*CTUW;lQ4-Br_Qvr-9I(G@GM)r3wz`C9Tg>44Fq zk2WI3VIQjPk_s~qzE0|QzJUl=-X@dOM4-(~gTG1s4)!x~TepUzG+M(%Gd(b}Ks zFvfgW1yB&IP!FCdR)N|TF%!0J7->$9DvUE1BW(yT-(sEiEj4IwW9V;0ds4Qz_H&H2 zu%_-ssKoL&rF!~B!W(>%GaU>4!qlF{G%~Pf?&j1;HAU|qJZh3bOYUF%i=a{A)AT|m zU(f=&f`fJ0yWzii6kX0*>C(LbPXZFTlg{OjisMIi_juk0(2}-*GTav~HpIzF^7PSO ztYySB39yZ^0XMO8)R|(iaLs%9dT6Ebak4Nxs^)X(2`Gp(#2ioJdoQ#1eeDIl{#)T< zMkaV+tm;nv=(95bNy+oU6A$r!pUi57u2RBs@h5*APPi!^_pE%nB%x67G}%AmGmdPx zb$km!C=*|^0oTG`N;TLhP&hl$%zlk{YpblIeZA9$oF|ObueRlh{ky$1)31NM z%ETberCIbW$^$; z`}lTV|M(+6s1p-PfBe<~JwHCSWJEr|U^swtXLCrh*#~v59ij8;!XKjevPt!`;Bh~y z8BL3OM(R3xjx2lhbSm-*jM}mRy&`j|gf)<#jJ5#M}!7Fs^vSOdOFkcp`^ufdNWbzzk%S`3}! zglN*gS(+0?#-T z?Xpa3(yF!UxPa0i*8jo?z?yiR%$4;~xo}si5Sw|&*9khys8@JAdFQq06PwlN#GL49 z3pr^!`+uXdW|DW1pK!kZ`lW$$-K>MtWYeasR<5_vy?0WsKG(u%c)6d@71>>jU9SJT z#>QB8daGT1lfco1(o)eLN!};#ycZElOy6eWMra5Vo3k6Q%jXN4pKlvTro1`!=2r80 zefxdFv!~uc=x6(DKl{D4%dZu|K^8b%sr#xf1^a=I)_+4iS4eY@Nx*%4V2k^$A9K-) zOB^O=z+KlO)q}zjM;+PnD>dUmwRR!xJ0ottU)`Bi9 zfJhNt8x~*y$w8Bq{x{1IP}F_D>c%h2971mtDB!I%19Y?hGYQ<9y4QaZGW%MoKj{=q z^g&GZw6WwoWmz0#QOqp1x8+#by5PY1>jLg(dlcR(G2FaY+xl7hoGHJXNqev%KcO?R z)%2=M&h<$r*UI9vG2m2_0{>ZrSKHottJ&@{VyykE*&KU)+njMFp{LfoVY#EUrIopS zWuvddDCg!jjBGt~ZW@kqWqc_`_Z&{N)h%mPyMdDso)7n5T}e7Gb7b|@00VAPtKys* z%rD5CN~q4tJB^8U$JS2u9=Q0-S@#@En|#Rhb`wxX8sb`&`$$L7Tw#4YZn zL=4{tCK+TK%u}@N1xz#ukZdf&s0Fbt%1+9xh-W z0vor@^cXdTSA@{rCVw}3^1EfYy!so!;pw_6xRpi?7CI>TCweS%muqEE;)7|ki5C|( z#orBG&;(rdlj4ZdnN8fzD6f|_%b0xj;D=i3cU8N>;|qb-q}i{d{ioNf8P^ZuS#cdU zLn^oDK{A~>NWDQ|Oepf%$Ij>64d>TF48ME} z5z!hn_*={X0NP#bI_!{F|S**Z1mjrZpe9v3dS%3tF9&3oWFo@ZAOg%y-<6 z`)2{VmplS{+MV-b{PhgONO^>towF=09lr-OG&{RzR!2?VQa$o)0ic?3t94wNOu@8t@-iUee9R?Nj#<97;a&uQJ=G})e5{OUfK6Uin|u-+5&RjK8({c%4!&y z^JSZ=(aBYE2UJ({ATpms-}WVCc=1rJ?FY|jC!#d;D%YWx0Yp{mgHvxQ!O5-81?0)= z6je9}lRMsdAe2o|*Cl{;K_K@s7PN~PGzry~|nFU28 z{1qgiji9eLya!wEOCMp^@XC?0pGQ9>G9VnW=ly7Vcg+;o z>*0HMv2<^Qdbv7s_UZRv^vC<)3OG0TKDcoYl*siZZ*M)$DbLP7^DZI)e0LOHtFawC zzwZ{Z5Fwa5^3+k0^i&WGRL+7P^&C|n6i;k*Eyl;`lxaw)qGd|BCR!_(CV2D6!n&vg z@81%+Z^IwR@OWrmugaJn7Ba(+KmpqG-~+jolI@kCT|mw{0_PGr9D}w)*_XCSoUyQt z@>@rx=~M%w6!yB=4=5RrV;o_|U_DmImYZF{OOAzWqu%LYPlPYOg86+!O%yB#ev~}3 zkB>J)%_AX@rBjr6L*$7`47R6ZCfR47o38PKyY71Q!bzvhVuZ}Fn`%kl%R3|wQL0Du ziEmk@QmjeJB}}Ub+r6&dj|%}Ji>o;dVDrV)C+r$#kK$9*Sv=6Gs6i8BEY!#M!)X`$ zSuOG)0evsyIRLikX^smm1^^EuS!$ycV`I?LOW!=is(%d1IwakNYHmL#n^bfx5!9d& zPBfmDo*H`S^Bi6pM@{AJs3|q3=5!~4l5qkzown8hsO)r#;=(JG=tgch@f#H;<`63S zn3)```ZBFpC_{q&lr|_HLWZ=PFf&UL<$nI9pk{yJ%*r3li}Y$b#xVMa>_bx_BrTGK zj--3@o3wlI87~_~y#`duE!yhj6zGIOumx0{;iI=wj+}8^fAN;u&M`+-CPU;&xq^hEH^h){hlo($j;5%-)kDh(h0?A1DxW}!OAq6C=jpowC& zIjM&ssALHgBPCi7&Y5z5uQWff;lf#21m3=s3W6=VGz4wX3Y>F$=D%3Xf3cYV4OmRdryqy0AP!xHUPw@x zQ7gcBQS!d}9_Z@;6#ff5mh$dt2KEL(i^Pmsx)G5!(d^SA2B;0Kkwl25I&o4_^B^8i z3BiThhRnkYroICHOdyiLSh&)KR7#E5mg!ZQU1U?j6KfVqN=eCPn*K1G!m3?Cad>nR_{vqUlKLJ@83jC4w)c48{$gEFEg$sgXu__!1@~7qhIkI-$ae`)xK7x z!1`u_omdKa+)?|N2S#*g|5#MC^lZA+G0c20MJ#{vfeaKs)Lquu>NC3QI2z%mX)bJ+ z&e6PHa?>30bs(dVxi|#X=Znfm!RXL^jSbWcHB@FR%4RSe!U=jyz9SnICF9;spu5Vf zN7qYT2uf_D6)nr0uxb9-$=rTHMRUH@a3YY7juT$1M3!j?UX`ve&GEr-P*E9&LGI#) zhQ4Tauvhm;G-Uh!gHWPO@91}QG=z1vK)2^LbP;rax4lEDoj8yY>US2W#sbi5aBk4^ zE3Bw!f8&Li>+YdBg+zg=-u-U{GCzQI=BHD#up#RQ6~xX6NtQ`eF!s+8mdmc_%8Kz- zBUEg0jr#JC+~x#s5gA_eMZ{BFQg4OPVrolY1ImbryHArhUV26N5p5k$19CZ9kLT_4 z^MwlN9UtYLQT(jj8Fg<(+LJ8*y3JqaXLY`U{-NWvKt{g(`6?BCG9IP%1%tQD`H*kw)wf2NwF%A4caBd7|-z0!ld zbZK;|dc&Z{RvJ#j81(NU*e$M79NKhFELL+UrN8Cd5xsqnF*k`5AwpG_!V8J4MDAcM zfsv@FIe!l;d}GI@5Iz=ohL(jd{WS*fRBq@2l%CZaB>mx_b-YD==ShL`e?hA62w&1M zv85%*-PN(Q6YB$!<+6f$tT9FgXy{E8Jnh+F8t&;&NIXrBoHbT;Ftl;JPxI6vtp%Il zXLD8kkPl8F?7Y}ie5a(o$)^sV0wGEj8<`h=dYL)3Z1Mca2yD68k6N2c-eg%m#q*1)M^Y+P&K6oc(A$=vgjzp_0#}x z(04{%j%L0Hj|F8183K19dh$$v_~B6_=z9!NB`!aEP;E(Cu{ztgMra-K)%Uj)uru{g zJh`L=iXSv6nh7gL;;OJJ>NJ(R2mQLUe6fTHmXBd+g8?o3+QaZJA*TQzku}8tu#h^v^uE~EIh)i1(3B&$`D_`KZKU{*eEWzeDI$RAb z^Q{8I!;Si|VLuvKhOI=;#kEnc!E5fui-%US>yi05q(o%ylMOF2=GDkC&EtnP9PvnD zZ=~di)|9ghKR$}8;4eNX1tJ@PO=`VTSZ$I~lrT zYJ$!TsJy{s>M1|0K-%Es8=|(aRO^HjlCMm}WY-y@Jy`po~S|U1*T9~?iloek@|$s z;pjy*lb*i|HqaMXY74i>;gSu<`&vhb*xv$|N*qW5M|DSYkYcJUGTJGZPpRMP;L;v3 zy-xo6xGih`ciLz92Yjxhz6h0|*NUu(!4s17H96xTk2jPFOzW!@V$6P4@r%j0EWYO* zazBiefk=%vlCYVzcuf_pqeRtt^MBMp4yb_l%rJDo2pW|y^DtS0JOM-^{Oq$~;3?`P zA%UQr97OCn9wYkjlLK8cm<T%*I6)oGo~k zA7Kfl8sn-aiLTsHX8W_Q`xH#*S7DBV79lfLZKhp^I8bjsAwlkBP=`A~>rxN&3KnvV zjLy?&^VjFPqP7Io0=yW2lA6VO?r^M4y}^R(QrtqhOvrv727#p~MZigZt}bMMxAtIJ zYO*1SP2}LkvG&1vN;RS3Kg@TEbajol!|c~b<}Y@7Z|h1=jk6d)OEI(lc=Mk#Y$GHE zbuhp&e=)3hP0{xiOW%`~SXo3iDPJ3xl*TN;rClgzyg>~_DMs)agr_=CPc5WZ%#emo zD={C5Ic5{bCpu&zlJ#JiQXv}A9;Uo;$c@OUu&bl5&B;(NP8P}Tby$GRZcQeOqJyl9 zePCm1#CQfpS)pK>43<8@%MpDLD9i4OT+o`b&*wicntI8Y*w$LA#PHJVP*$P%pimGe z#rv!haoY*86xnemo}l-1xJrHQ98Z*kbC4@tNW?UqL3lOD?`JYe*(*_$O4p2^wRHS2`o#N+onQlg&`_k%r?L*9fpmyTesPV^0kfI;I~#O-l<{XcaQ3o% ztO7q5_xO*Mgm*c4O?c>1sxuDPET=TJ$uQS)9~n9}jfV6Ef!RN5o}qNv*aT}=i7i;M z)KX&|O4!B_HkR`FSnw0B-xF}Dh4=t&_muOlKc(?dSbZt}t4s=n@z%o)Q1@*bI)>kn zehoE;x41X0yKl)ZZuKn2(&!|bxWlaPVRSG??ckTAL@w|d*TeA+uI*bQvyO#k)JGkI zejrCa3H;)d{TFV6?F}-gJ{geD3!sZ#{iyf^gqkOwoKW+V?RyXaP$r@(Z-o z>52A`oLlc+Pz=KbNH18Ni%*6d2@R591?h(9>4GDu}vw76ct5J39%&L6#f8A|Eu zrIk(|>E@kkj9}g3Y-uGrP5B!0NhGc-o$-JjMaa-VayJdWg#i6KdtIN3;Ly0HPz6m6 zp&t{!zb}>0&D)eTU2@RH9Y360$CgPnxH$yz{>_KE4qBV2mB7eahw{w>YfXId`21&u z;(az2e6v6;ER97${9P^fQDT^xNm9IE!X-qd!_Opd1@Kl_lu@leI;JAoBUAFG*H_z| zEXC|*{WwViUEAtA82lXhP`kk?`2F+{dA&wZ`4;!2erY^e%J;vjeAOuadM{-@nbHDI z@UoTixXl8sz?2uf^FETxU#u)8&zdZp7ZioD11_%hC#@ml6Cd~fn1hug8+=Q5p2pcj zOJJP5IN5`WG=A*va6)IzC`%2t)IlckxRmivC=NB?EL!Yw!rJO5MIiD0cru~rl!|va zMWCHl{Yut!fMcsum`8WfbIt%A3Ql^QW0^1-^#{lGC#>*Ls~SUDVk$$_h9_(Ingzj8 z8)?fb&0vgT5LsFfN9gp1JT_0ihPR^iVAdYB4xDO_Lq{wetjy9ca<)hjed@?08A`b$Ni@$ZV z58G4=X--|VT<`_+cLY6DO6QO~i?SL$U;Q+ufe_Lh!o5fecKXsWm=2(oLRc|0$UAr>oA5LP z=o_e2*yEC}cM4`AKm^X17^wX;_|g&oG2S)d|F(F{P*%>Sg2YmLoX}-GcM6f8oUbB2 zHCN;BQ)oo4{?mZ5M z44PLibEIL#@uafE;2Ce-nRvYxWYM_Vfut1{XciH`wX~bK57TqjfnwjROx-ow-9whJ z>iYA`y6KY2pfI8UMj<#K0z_I%Rv8BJtp+@P18479bYHEivtVo5fgg3ajl@F*6XQ=0 zyyfJBGHFbAuvBcmw)`HvIf6#aIks@KQ2!fm24U9W-OaNnV$2~=9Z8&@4BSP)iNMWK>j_P zA(I8B$lpj77jjSJWt#Qv@UD}61irAFbW?j^1*rd$LhstYOQ{~ZNwWi)3OdNUC1aY? zM)hT`L_=^EY#isGWFm--IbUpU!@8xGiEKw8jJs}>`bq%=ifXK*)y)kAk3>Bw-q~V_ zXZ}hD8O;~kC7xJS^@;Bl_n3rJ^&O!7sF{$7tUDM^{J?K=z6<*|vKUpJ|0AG*=(}~$ z?(oLQ!cu$QA=dfmk(J*#d)f%qgc5fu(PSQZ?|qrtj7spV}TAGF|v=ON~{48nt{caJ`%&y3hn1^B&3JuxjHDS(w{5hiV=W>lsv6>C0{gD~GI!i{n zB3l7 z5PJFy5|&?`{%p-Ntq?D~gqYJHCt=fa(h<3G?G;0^qt$J^_nS5D|JAUofiWOYBTZxxkzkoqG|gcwMNasjH-h-Aqdy$2 zTnsPwouK~{JW;fPsn1kH-HS112ATCWEZyGLsY)b3RB4Efyt0~__jV$kt0|iS?`tMw zwL>&qfd7wB4xQ>-!uwaV8L~apmCB7Tsk{$mQPU?*V$3o=dW`ci!<@;5#?|e$Ev#sM!Q@lGKpoXHJX|NS2lDw^iI`UA^R2XL z&#J}}209rPCn$<-A88jA>|dBBNQz~kGZZRf4Vt$Ph}CQ}FIZj2KNu7{;x34NRCG0Xx_>=F2%OTY%Kv^=r-JEuA)!h8rAg~w z>F>fGsRxi@&#wow5YA?M3mRm^G!0v6No2HCO4}NZtmFNW$Qy>F}ow92S`Ik zq~Y*hObF)r$FOG%uA$<%qw2+39uYRuG0A}^mDpnAr{yP&!9~AENz8_%>?3}zpidoG z{r0Fp6bltn2cWxeAI}IzlzE~M)X88rKAvmNRCR`&!D4Mm&=akq$rwC)Uy|CWm>L&H z#hnbCz8-i?%)LyH^Ga5GL$I|~Jb?{o2c|ia8_xtpm|7+BVR7icuJQ>9J(F;=Y^$yH zv=^_CPe^Sz4XSfd23;6iv^Edw+U*bpT0o31@)F&Z!510?hZ1 z1$`QMz#^}rcVwGmVE2C5XzTH}$U!jF8ANE zzgCBi!H^XXWq0q5ustlG_Xfpx}xEDge%Xp8W}G{ zJOe!OkuIn}Wkmp3kp3Pauz)(!-&(w2&QMw-=G{<*L>)CiCi_i2)}$&pN=52!HpQdDR%{>sNbBdC#q? zVG+@_UOpQFa3LXacFiyGb{ zK3KnULaz^S4uaPX=o)fOQs&IofX`7Ue%!5O%Yo#?^J)I_=M_;LQRK`sf>#uvfuXGb zk%6a|)2d5uIgJct?$Qhjy|e=yXk0NiW1M%EX6&fQy-3`jkAS;xaV*$1{LDN_Ul37@ zady*%eG({@HWu!9t zAP*9=ml-Zzy&yPwns=?{)QLN(Rk=L&{84b7xv**Ah2;~_g?wjPpm)r*@Q1)2iL>a3 zQo%}l)rEYVUlsW!yxv@d!owFp(v#SePwAz%g8m5h(6M~4#x{@>>oTI5(_)-#*Ev~z z#Ab=UaZtqfF>~95?K)ma8;nB0YyzaPIdK}PLN&b=W5P=LZS9XhphS;^Nxg|7y~JS& z0ivo^W$C$C7^wZIepm`tx2@W%h;LQn{Mx)co|)wO;7sw5ja%hq5;3xiYxCL28bc(RJgm`vSp^E6**_v{aiYpp|{x3!I6idLpoU|525YB z3~3!Cze&XbD`t(odIkZ_e+^uF%5R0MZo@0|pq_#<<+nG9#H^Xc*$YCa{beSO-Xzl1 zDymgorvXn<0Ll&~R}HNmcb9L>F(~v0#QF3{QM)03@1iO!l!|%23rzpi7bA@F^k3xS zH!oM zvL=z?t?F%LQA3}P&?d~g4lZpbV2I|z-LV&H!$&wojQe%mzQA40*P7H*YPBPMgmv7( zr40XIv#w$rU&p>@3zFl~RmzNS-{U&v_q^4)E$`+0xnV5|oIWy7ssVP_@!93A1NJ|O zp~4$JA=0+20+!c-2wc}dN(ldjYNgv)AC|Ai zpT-@rOurap7bFhAM>LF6qsKSO%AUG--e?X#i`~jXXV;5quepU1m~|i@zBlJhJiJFn zy#g!1-I>~Bp-c=dX?_h;NWC=R<4Jig@UcIrXye0=>#j-3(IGc2Eq0oZ%NTZS>xl>1 z%u-$C0gl^4zE9NUD(lQ|*leHRedm?}i~bccOGHb>im$y;GS~H(mC$!lew*KaI0DtrZe;l znOKm&b`JRdOtd)<#P`stbIDm{Ol|P~Ti0+kXPI=TAD$9dV?*ykLnk0zC#F+-ZrU!~ zpcX^!O6;00(}3h!YU_4+)iiF&-7Im|(MI^(dq~+Oi2v}-Gx8H~+yQrpV`A-|zhAVc z)KZUW$T|(fBMlvO-Q8sVVCmoIH}o%$8ID^m8r$ly!^H7;)kP3F!9T4$URqbK(vv$m z?JG^#R(%uzPM3EiynJ;<=bHHDe;?dzuzfxH^891kNTL;B@>Ov{ojoPSR{@U2!cbOClYOC5Og0u*A5#y1?T zs;N5Y=G|NlnSmoI!ATnWndmNE6^d1(QP3b6TJnS>Qi=m08&U+k&QNa1Dn3ql#wUd`2s3q=G7n_R9L{WP1 zd-GGg6Gp1aI`Uj}WU)tAEu~k1x@6(6{k1J+-)ge;8w_HqVbE&Rx>tXq&dlBUEa=`1zA;-b;^Gv7(@#Ynp3!A){+XeT$2d4xXpU_fYhK zeOJezAbnlmiUq)nu!@1#wRy|#7C2))?N_8L35ME()8NkSkT^EChx9+H1!~gR@ zOS3&$YvV_4jOttQZ8toEZE`#|3aYIkx|G0KsY%ebIjj*+mamT~LwZ=h&@C8-M0FZ* zR=&MkVgIr{4p~8nS>NgcfDHe+sPi{L;U#fv8MN9;2xsT<{C3q4DnLKlL)lbe5Ew)2 zl9-tAxD^=7@Z*|#@U*!PK`YtjbLeldFYj`nU0NLtV>3J@&CQ91hwz-y6e}{NomR|* zx`b^9YSG_6I4|qbavUt{>#PV@6BpWX)}kjNN*d|HHHrXqM#aNMhZ=ioU_uT~6j??L zRCRgl7enxOQ!&j&IW?~WUfaMts*(xuW=TyHruy_-W5-X`H)w_;1W&GLtCSdd?zz%= z_}Z+(&^i(KW81rs7IR?YYwTndA~>~u7Y=K;xJ}y>b*aS3=k)x3FY2B{QOb939FR>i z)#Q+f`0J0B54X%EfRZaS3F!Z-{Uw`&^|NI7;_oEewXH~$&Wa*>j2&KaRx*=f5?`0S2ExJ?6eVeO$dkbIuW%al6VQyE)HfXs{$2EpQ-n}|%M5wxr@k@z63Jypl(muTwgdHdb_?)|I( zn_fEq-!*Ra?qOY6*7feOu0Nk_sm5H|Z(VzEyMo7PO-W=4%;w&M%b0!Al>;okBDbnC z+^mhW^RX%PJ_BO_G1>) zO){v)Z*6nE8W+qm3Pg<1o%pwj(sCCtOJUf$H9W?1!P=uj#(q&PQ(UfdnYXEm!}Is7 z?WLJvq0Y|@%zK517?ftPQ`fyyh>a}-b-p)y`cl^beX6>tJBD%dNr0ZxM49%)WZ=m3 zza{NrThGpawn1B5?fCo9OtYc&_WD^BO&XU$%VJgwlWZyUtAZjoTTG?3zC|%E3*GAq zcJR~VGWTZBX4Et2mKIfZ)lbP4ZstX@{pT06>rg9?iid`}jy7$>p&c&hz#KuoH=t21 zPY#KFGx{tquy@W<>lsKyB+&H;Y>zm;x9}*~*Xlhh4mDYL>$#@AXB3An4q3JZ2^V?; z?0<2U^~7}z(!tf+0e6IYGHw${jr8Wk+nSoc%VV`QGPK$>j7Jz-phc2w7- zs|KNu3bdL?hu&aahQW-7lBd>IFU8(|i|y^=8Mc&InWIr{2deKekPl1&qxF$>^TNiI z{E?Hq%+?O+?UMkrX4`cwx5Z6PvE`n8hK!5u-^6RHjC~i>1#gVYtRC#nkbTdbRlwSY z+4KXNXTZ6_muIYyFScjG_aMrj`5Wa$^gm~b34MGK8vXtU^a=p&_g*){VQ(0Y#8aZk z@9Wgrf0abnAu(ClZFhcy<##N{{WHl)M71CNhn5cO!3#E0XdP`?ewEE*!6|=C?Qsv7 zbk<*0#jw}7&qgDg*s`bztYL3wwIK0&$(``4FK6yAWPkvvt@^q&(3Nq&qN32R=P%oj zvMdjs10lYzR-aWN^8?M(tI;xxNANOcN>I$^QCY z**Pdr=SpS(Pg3lF#ccTP+%?SppUz#m++y*Rde${B_1n9cmpfkB9aHUgXvwQd?ixX! z!~7}fj^JzcT=;$(Mk_Msi`Kx+!B+O14A~FCS6X^1M=`Oq1>DlXsm8=9i2R?rp-g*! z!HEvTba5H)5#PLj#K@UTqkXd2Y8&rst> zwuiak6}fRu8B1T1Y;Y3KK?On=&BkCn=<0HR1gWl{95SD_`ECVm%vQ@1OIVsKvm>_3 z8A3pl=_QBclBo|8G0_D>olmw5hH<;pI7s1;&39G>@ZO%c-FuUcJx_pV1Ssbxz`0L} zuQIRELESe$^cf(-8?Yux;lc?)PvCY4Q-_O(l;l>~OC#K74BnA3gVe6#=~0Mp-fz;t z(+4cP`2$#P40HtktNC1s;m5~2_B)?KuKoOU>-B9{-aua&n|eu)9Zv&7%uWmEH}OW| z3erxJ<*HEv2wH2|ea{TWGaMdunt3%E+S_m2s;G|6m)DoJ^a<_a!m9W?=Yh>K#OK=b zKuJmD*z*VAOF!dhZ^kh|`lOz@{0&~glRqMW$kJXN^S92id+t0pX?GbFoXdok*hI7( zP631n{XI7#t`!p**klv|v}C{XPX*FyCfH+!E)x6PJbtf7g1bKE#=r#b9P?qYwH<>h zKNczKRO3R%j%4v+q_Liy%0kAO^?+&OIA4_ai79Kpg|;DRQ{ugqCr@bqsT@yd*Li-W z^dg-7_phVEXjs=aqAu3BS&!4rYKF&Z7e*^>S03?ey)}ZnSdLbFw`Sk%C1l7NOLV1U zk!b{N<&gz$jADk|5+lqP-C=iH-Z{M|`Nvdn;c1oY?xIbwfXYtNqJ7U5^r|v9DNkZ6 z+-a}t6wR21PT^wA)^66s8XPMcJ|iX5)85AMkxNC2CMq4xgE8^<%PWUYs}Z}IZZ^p2YErm}1>CebF3vaVmu!E>Z8ygUZjFIBAZDST zGtd`uH^(h=-6n{MY0*J^LK-uh!<3W&S?$@B>b9b4l#jMc3884K=pH}E|aQ&5hb64b(*WAAa+gXLEgmpPvDMR9Z z$)5%3Zc#nt9;&T_1(x$y+aL2}k>n~6H}LaY4Du9@{w=lfYe2oebW1s&#MKIaDN55u zuZpJCR-cMv%Y4q2Q5AogDcuw@2NtXyk(|b+hQCvPY~z=!bQ1R)k=$rzsM`r_yq8WY zd$v90h%MU`;x7Jvn6cn>H8Rpxvf>c~BPF9wgkFYzeUnid>-))W2wl_mJLjy+rP}D( zK$IC=z@TRHtlHVu8p)8#`B&86_69UAE>4vp61>f~l3)K$^I<1wVY2Q@WzIBaY-F?r zTB*N|^lWPLfrhpZzXZ;&{H{4ZfLFvZcKg81F;&&#y7}Uj$BUZG%MMO``nhZN<`d}0 z2VY&Qp_tJ3j%v?ZX}_(?xb3+q$*PlmS_3n7wY@9-k)@`qsk!nZ?3c=?GXu)?^zwyd zOE=N~LDxG)XVQghyRq%0W81dvj&0kvZQHgxw%M_5CmrYg^R2bV9{b?mCv{XuHAc;v z^Qrs3uA-H~?I){`*%z{=zC&|~_j+MM8yE2Iuy2C+o8|ML!6sz;OCQIBi>O=B`qys{ z<@T(Lzsj}duxhiKBXfg6C){cwo(%H12)kQIigau*S@(om#Qu9HmRRVmqkk-Dw$aZA zpKpc~)q2v#$qr!O-dj&K_m{uS7NgB7;%$uzTh^5-4=gdb*3**5)ik(P zAs+3ojvC!-lMLCj(roq9ETrn&vp-l{uLSkFfZ3@pC77_j)5uxiG#!jo zZf(#0`4_rTv8s(s)?2%XJ?>$@r&YRM^Q9U}k@Pk*$mztKZ406EUSe?l&gp8rliR1k ztr>CenOmC1Bq7u1x-P!lR@f@@qJxRI6SLl|9gySQF7Da)cxv@YAcpW2^>0@K@#3D~ zMxdLc>-Y!6VM$S_{GbdIfKDWc_jaS*xvVpAe-vd$u`SYUTM~x8%u|fVy5#e>;4104 zm~MAW&g5Mq_+3aMq)^-NE1O@v7|^tmdIqpy?b2@t_%Y({{CAA!Hr+Q^ND04U>8H*3 zubz8PIE@kfK0z27ArMKv@64C@d+JAa#y*b7lhlJJyo+&wnBvupEbVOT{EL&^cD+HF z;FBg>5YwPLwPD>Fk#ki$v>2qtvtALW4RYRZDMn7FH=!v4)B0fEZqX@E%>GyGs-9k{ zZ_k|-8rrf9!=ac>ex@9lb@>sDY1&QF4!oVi@~Abh%RIAmY>U};qw#78 z{}vbn_+VUMBe<5@)|x#Mxmq^UAIC6P`qO9BU*%*p^p-{mD)zXhvk+|Rmo%H=7-^T< zvV;~J?t1kiMd+g|8G=p~`mokWf3sb$tDXL|#Xug;nPw7Xj@G7MZnQ_;L=E!FEJv|W zYA}pBxNkyaBk6S5W8u0xa$ioet}NOVQ#bl>YOAdM3e-q{RP1SRbS_q+j7?MBqOdqB z3_IKoM`RYd4oAFYn_SsvR~0Kw9eiI_Z9N4F9i*vzFh(T6)pWp&nGtWlxH5m?j9>J#^eA_5Pyjk zv}sV+KG*hBA_gR(o8qnrZ@EkWU49CLz2MyU#oHhH?@k+k0BR#=ZAbW+U3%ONloPAO z9J9nI%#_Z~c3YQ<&~tdnzjq+c6%0j<$}e62vR((PH!y0`YTz^m(b(;DTe692mhQZEf3MsS>`kys6do0*ezZRUD~q1;P@^;=>8pGCO$s5wPe%gx2sFA>hX7OalK?r zQj4WK+ryMFZKrG4PSBoLNSMl4SMPk!<>Pb}JJs@7Sy$&c1Y{!@BhEL7O z_%*2!u?&hm=V43$H&T%+HSrdcS}RVhagiCfv09cV*44_lzu~~5>M1JI+0$6`WhUe0 zI92uamS6kYXE2*2#w*htWVV)9_~yI^9LA^;q4v_-E(&X zaZLR3f2lZ!6kLNl4xa%{p|AISN2oYef5iiL1&g)l{<8iD_$u~({u1a)1=yXPw})B) zGRa^cRA_(Nv55~!OPZD*e{%DrUWJLWv+&p zNY3^4@GqpyO5@BKfZsLR=ZG;CWP&xqs9)i8r5 zE+lOCg!PG0)!h?RAC&4*!1~!e%p~b!1B_T+_}}$UqTR87_3*)mL(v$vCsteJL>57y?p2Pl&4s!1{FkUxL~S`#8?UWrBl`-`+15Y2ct!6Cl*>VrZ!Lii z&L0Uc|A_Ez$BbJOg9OX_AwlDw5&Om|H6-{-e>6qW>1HP>cj&=1|BP5+4OkwY z>{PAck8=<(pW#Tjv+%Jj-8pe*-P3Rdm6d#iJ|hlQGAb)NlM%XJ2vEfFs8nB#dG7bAA<6gH+0XaoCEL^LW{`-^GHE77 zAR!^{fYRtZ;|Xm2rWGTw)~`}>XD41^oRZTE3GR#NEifkNF!9y7zSU}Uj5KoTBKcXB zUg}=+L;NjHJ!7**Fb9rb;#bQ^-$&hzm}9U_Sak zfdA|r6PGa=g{pi-YG=5~?>$!epW99gcvrvFF#t|451(IveepbgPjbEw zG6gF6-FN=^tlbP$>Hop?#%I2TfWf~>^8r)HS`3Vn(HRY#J7yv>mgh7_3|nRry_D8U zg3PhZi11>K+(7?xOp=<|y246WGPDP}6)>iv5sZRlT1u7AGu;QL0dvyJZ)u>a?QY%( zrj%h4FNXGh-5Ut@E{2RJh>yb;;KvEuPf?bR4}V(tx*Hgx@@B=!XnSXn_?}wuowFV% zIJ2O#w8>;VYIVws#{-mD!~~}7lPS}43EMOBe>=sPjpX#8xu9CFUU>OBfv9x{)#Wsf zkfDNj{|xM4PBPQMU*E4p&YUq4Cz@v%w+DG)kb7KZmCU zQJE^+4Uy~1P83N`aw=q8Kq7C3ml6gy1c{jlRApdc0`nJo(j%p84~Z&4hBpyt=(oZ@ z5&XlQh>Ad8JGg{P0(*&Mc-L03jKYIxRJNTIQTaYtkxn8j{J{`c!Z(-wP6(P(a$toE z!$V4~oBdx5aa2c`rcy(Li^0988d+jL@Bc4~m>~p1Gz_us~hx6?%?KY|JI)U#Mo!Y@%VWCY?fHmqOXG$Nv(8-R!G-D z%8uRRc?d`I*p3z%7Re2{RUq0*(>>9ME{Za7lu@Y>@H*%6XFkA#5IW203mu+mtp;1uAMRyhiNl@}pe331tGz-%? zehDmk;WufE8dl6fa~3jHnO4TEGc$(e;0vpwBJQL_({RAK;+dvy&}Ibxj=w50Ed?Py z9vEhV?4^dLDb!Sj)b$$Jg6yUkyOfpF98>iT+kOjk7b4% z2dw>%%Dj-UFMvNKMOS4Q0y9L1H;czVDpq<{r-j1--pfS==i#mT)KYq}&F}fMU^i19cl?CNqgKtGI{R5ZbaNNzM=z?dr#D+Bg_Zz11`-aQJs%&hhZet3^-kCnrmNgcI)x|~)5ADMh-#A?0Jh0#XYJ(> z6YUX2Yp`}ei!#roTDXcd1%akvJ=ZsFd=|RLKS(v|m9W*-s`Nx2tOxqab#Ov;lm?x^ zu7F~Vc+k z3nYxQMy9bhA*xh8WFDZGcapWwa`qA?27s_>0`myOI-bf~PrG1n55>iZ|Gpw0$}p1v zM^q^lspW57C2U+F$&(1aq0MoUL9x*M?YB@l@<~n<_MF;wD;H}~uD3p(bKu7&Z1;|C zwo@b3&6KFqj^$O472X0D?TBt9!8z==rn<5|DEFJ&i7NDlFE{eg-~?yZ4G5TCuK)3` zKg$fyDS;lTDb5lY&JEsM!3n;3=;(9{5gJIm$>PF}WLabWA$y5pUQ0F40S6+$%$5@7 zgdJXo)9vjHu(=vMk~MSKLaWxM8vvs4xe;%X6g{KR`!-FPxDI_8l0r>@tH$OXMcTz8 zbNwrZsWIPcBOXEQ@M%aTCzx@dDCLa3q#LDy)HL^tx%v~~PO4`-Z4taXy=}N`M+~<+ zg-i}4N&Hh$t|{lu3Kf$yv_T0=qhZ2xfHXRkS=>Od4wg-RMvaW;{_N{9mb|A6Qnf|)zdL5q*e$F11l00(@^=@wQ8Fowsf#8$n_Ibw z$YVcEZDcLhuD7(p!{*=X=&zPmjXohYERZZ}U(EKWU_?sc!&JawCY}xt?qFX~!G+^# zGt}*y$E@~5#&}SUc+)otpBa70VP;%bB9L>QEDWxOldn+6tmMo;(7PZce4)wy%nKBk zbWlq-RXd`M97XA^(U4;gUcq+0zS~>xrY*fUU9VL^tvxGKOeExam(e#Mp4{C>)&mN9 z6fZ~CX$SVlFz{^nfl)#sphwx=1xO6noQR~HIXD5=IGlF_x^;=^0-WC2-ALTktRRO3 zrBUh6YGcD>HB%BKKon}#wsW-EaR?Ja*7`X%JQqt+_&|4pq2q{JfD+OYv5Iw&^KHzQ zWB+2zmhhoD>l1zDOV%xcG6gMot6^es$Vt^OHvEX7-V%?KOp~tb0o7|Fd!FN44}^wX zG6@^q=Li~w2W^O=O%B^gCdRBMXZMIyKQ=W|n~s+Q>6~m7`wFvJ>Zl<0SSCAOrWu8P zZWdz;*{0!xi5%lNo%Fq3Os&l;m53A*UJ~56D7lGRo{&*~c-k91Vs4zjS{wx@8FwGF(t(*D81^M-6r$psisV z+|l_TQq-BoQ1e*sKmMB^ZKs4SXpc!Fv{$egh*#a<>6CB4Olo5$ODi3uklKN!l?yEn zw9o6K4SU%iQ={I8FP}9A06KereL2aTq?k9)t{5thz^|7*i~1XY|NQ2wKksaF=xnJm zxXw#5B#^oWetik61DUh)oK7xpGh0;cvRP$ysbw5>a}}n6#{T1n{j*2bX&GKoDDAH!{$%(tsc~9e1!!ZmsP;O7yO@i=I-4ON9E1DW0KVna=bm z25s01Hv-FSs;5P`PaV#wQFJZNr>17(wsgM69xN`VG-~i^=dH;C?^>J3s^>fOX`eA? z7d*gG7~SRy7KcNx$`~bIE1Bye7P!UEtwL6YbBgRXs0M?ofhqX6ou7`te~35s-Ki)W z%xo*oA^T}nGwTXmak)djAi%GkG$*g-QY!QhMN3~iw69IgMz(!s_)-f-W{D|=$rY9N z0F20;l5;T^YCG9U%?=s4c1}1xxHd?MYJ#dJJK~@!-D9%vN5#8;3*qgr6W~7;4o72x z`taPS@ou71U*jD`bjrucuX)YT11IbwRR z(JxQ-n?lP>%X`peG*MsOwwj_FmV_8_tmD*dg$}?D=x;jT`nU-mwgX3=p^zkv|X89yJWj=1S zyII?9bL13&v^O12ZE2i##0?wIRP5@%-s3r{I8Ad|YlvyqhM}3&geQK^=5Drfg4y%9 zw#k-RbF?!*3i@iFH*qFRTkQ+2Kh^d$d9|SX4ChhFz|NQ1Nb)^@P@Z71(F=(bx=_*n>>1JGLE9#3zfV3$pG9WuO$8B$N>s%jo zSu>iH*z(A<+hXT*o|j|SPhM0M0+`(MD^pzw{w9uaS^iXT@dc;iBvRZqkCPyS)p{Cq zK&z!AQ;Qxe8ujL9%+I5K9!&JAej(3LtWJBLY_&M|Ilow%xZ zw5K9Ep4I1N1_;kb`^yUhA0#YR$9*|m?;2WZpK#Kn_d39ep9=2gtz92Z7Y%T^Zn*vJ z?(%eHU<~k%fjm>XBtKb`|K!@xBYi4%rnj%jqcZT*h$&U|6r)cH*_SXnA7(femou4I-A$)~(BVnFGkNBFuoD+vro=4B=bLICC4ex$u)qIZ7(5>l$!8`vV+t02o9y8x zPo-muOs{|3y}Q_vkrqjr7xhctI$I%ki%ahoH@cW2-7~8rz061my>@Y zey>0EW-laLAa?KX_2u+cX&nY^IzQ#`*@|LaiFrXKcL`n75k zp%Q&)W20mCl5iU92cLYZ(5Bzl*U`<*cd61FSRAEpa{oxNX?}VOc-gJ<+gJZt}l zPh^!=tlH21MyLK)qE0WLB{~U6)+|uCO)ZKjTf>0S&+*7FQFfn*oe5jf;Pcf+5d@kX ztrHW>JfV9!;vWc)6*s7FT#Pp{gDY&4^RHS5Quz5~1&aDvhI&6vUy_G5A4`l78Gi-- zuh$EE=V)j@XgYk34OZ#&UHTn>i{Hb=Tk^G1N=Q-*@Cp4xf3#T$ArQM$wqns-gd_do zz41`U9+D^Yg?o%2Yj?HF?_Wm;dee<{ta%V|8}H2AQ_xe1FY%Klk@9^7fLBB=M1cqu zJT5VyANF;p;*oJmac}DcPVa_xH&Jsbj!lsnLdqJ9dGNTr1xzBoizPDbiBu28PQ(4f z!|NwQJ-u8Oys}KGIQ-L_vvbIRiL{7|g9Br`56u~jFUkYX2C&1|EP{KFbk8So#b$K- z;V%xxoebT;$HPz!d$pFoS`lxiRvKb87hTO!c&^8u*rhUr0cHmdMx!EssSM8OCNM5% z{P*BBj|yQxweUNVD)6DQV=hfWYiFC%!f%c|a@Up;M{5LEDvF?wq{|~VO1A3drjUyx z7rfbRdHiYVU||2Gi3w^4izx**vHvmhpR2OaR;~Ltoc5!a|3P14w_!a0r?b!QrD-78E3JwnL0^ADSIv4|d zV^014={qB$Dvj^43f!)o8+QQM=^GiEh}3Q;0Qb56bnUNwX-k(Irw1JFp0DAvt9_$K3o;329yYtTD8K&S3Ja z)u^80FFMp(M+Js2Xy=>yt!HRky5_X1NPPLGjsNq@e_;N-q}$hRIQs!#8*OOKmt}IV zN?l09ccBz+ZvFu7HL}qWf56vs%Cgs(*fW-bhrnID$aTWRYV(;F4YkzGKYsb}?Yalw zj32*zLd_OA6nzrleeU8#An%&s+Wx3-%md#IP>d5+=LuheTsBLMkk)bv$x4F8>+{Ez zzZgWCe!vbR#P}0_Oyd`wNdNK3xG)4WPOaAz|V7T+!EX5`pS*t)kMAhYn|Zhm+vE_3;Sl(_dW*Mw+*cgj zBllS@;n{XX&NjQ?CX3R7_uK1t$#||gzDuI@+OkfsZ--rZuooeT-N5-h^wC1i(JB%F z;aTPDS(}vKcGdJL40`WE1h0vDlO=9`?&NkTHd!%p(gKWW`6gG-IMHnfoL^7qw>Es8 zG;(hw&UGc0ZJR!`=)*F-moUG$Y=}*ots3D5dEW&wV+#!Fv0dSkX0k1{C_x;vEyCqq$iA)Vjx07a(Mozj&hye*7E^qYW)3$4=h>b zEyUF>jThbpw(7?KR|I_CZJL1idz(U*oKsTN%c<4vG%+B;k zQg66M=t$$?)%N*I(FqSsQ_@jS7ny#kAN4}={JJnSc63VDCKZ?xap7I;_wO-G&r(Ye zHZDm@vV@HI*Eqn#!3A+Swqx5obcvjhKVPSyVG7e2frKaKH~JpW_Z@*I=C>*X`5`?( z;>)C6s@=vS0(ADipJ;RaeqHzC>Q%(&@apv~1t8aKadq{%v@YyFqA3a+Ca>RX6qj%I zbW1vpCjbL|6Lo<7_2+Q6DIl!mV;S=WWe(;N@VR_H)n5N;*-j5AB>1m~(ZB@Tx#J!u8$pe6q>5;uUqo!J#vH++ zTe2~*9Tl%sriRi$%L}teu`w)3_R6A$&`=GI{JY_qtwfT*epe(|(=i}hIK?K~z8{w1 z8oH#OU9II<26ojIcaYC&7_b1%Iol*eMKRqgcu6S~Edx6ZMsu?qm&sDV@wi)HdN<%& z(F`Y&pW9Z!d{IG}m!I(qlAl!?MvB8$^_pYm>Rb`b3KZ%{C`pNTph(;^7&I@P=9qP9 zvdIPBbkE6ul|tVa#pO>9h>+H@3#uiZY?KC@y^OQKUE)b3&Lz#+ZVEgs>{|dK8^);z z6C9*A<)CPRHZlE4*d>fh1(+PRjBBEyfOG3iuakeTG2Bk*Ci6noyKW*=G(*n{CnYE% zYnt=(3TDl-3l1S(K+Bp5!k_PY>NL{H#w|IC(tBuQTbMr?^ZIufV7t4^^Qkb_IqsPDIH$Du z+8xcGvw&1d0?(bi3M*Yum1ruA(_kl<98kh_N$p>6CoQ7zb$2vSO@bgNB8-xfNYz7D zNQfMV0vh-8((O9NVGn*{sSdx%SVml*BLySWiUQx*88AhR!A)K-k%f|EZ*ZHa-C zWX>lf##2E%39(U4h_hzH;2#OelR*pRVgkeV+VNHC1$!m@rmVQEdrM?H2$v+EPc%D9 zlL@<*#hh*q()pMLd!vY z0+>hi5~p80*c-Sy6@lR@h*_F7KDxM#iL~js%%xN zmqz0KLc`^=4`W<;igk&8C!6Ep@5z31xF)h1|ixBSW zJ&jRfgA~KCdx)UHPKeA(5WpE|f+7WPD#m|+*DXYCvU_s#;O3hRn)WgQs8VsOe_DyI zh?5}w#!fR9JCY7iMU0b3(l7^NSAGGR{Gmi=gk&{Li1iU>ZQq|#-oozzy(voozE%_e z0bbvKv6n(q2pGUWT5y#MUv|?p5Mv7B%4{?yq(>l8X&9wA7v=AHsMPt9`A32pfeFXA z5l|-#~Xm@5V?N##c3g*&Z!Kw%D5CE7_$K>Y*>Bq~yQYgq<>vz?p}q~eM@ znAtY0v}Zf>aBbaIA`hpk`=L8U{wcvAbIKiRw)Ts~B_o*=p>7UGUSV6PU0IdBf*GNs z{;vnVm*K|)pX8iI<=)a2LNhi1jQmiYC7WoA3&e=hV1KVaskQGTZe9iYgxy0*J5Jo; z5XWDvPH@i}3;llOBaZK%@Y$#S1rA=h3z+~^y#49@X1VYqc=>QU7vp5Q}T^qDxj8I#Sh z+$Uqjit*Y(gjtDP*-lZfV5O`k$5|_b;ZOf#f|nTw1hrL>QvF~mf8@yX%mj~XM z8mlfU+(}Y~fzFP87tkVsXc1hBVg+foLP@@*#6b-yV9})Ne336%8sIcF5s3LA?lGbx z>H1YeD|up8t`@f_Y;QS^!7&m`AUVD<+Bv-j1jl+3O);<%;EX{rq(s2c z^({_;E_}kPbakq$4jyD&gG_Rx4h6xv@L<)t^tM0l>T!QLa2RTSGk!9bC<94=wVGzPUjglG%9pFaLK7pT7o^aHgIn0xdJjnY*?uskrWb^ zRfSZy8K6+ytfccDIZ5%DRLT=#x-iFKa#!7`F-v9~vA`mf{AjfzZG(5Q90?y5s%2r6 z=$QIHAz{+PiM#O2Kv2^;fx}8b&2%ao`3ukP9}>IEV?@S84~-W2Yf&-9(iVo{aZd<^ z+q5$kv|a=s4`O=7!J9y4b)cboZ1VF_$ai z=A$;pB;ABLLPbh3V?I{kkrC_EB%#h;GZFLDk}FdsNkTPK)}0JN9iFwWc-^3`5mkap z(8}X7J<=giOHf9`o~@Oq2|gd}2fPZQ2}V)ZAjK)g0X=?pfM?lb+yny3@&0(2v2MmD z+_I%aZi@%g_Du~XWv=zVLqMK?|1vtA(zY^)=Ts!>0+X&+E*K9Cch$K=HCoAZQ`W{W zVkauPGCT{)Iu2He8}fIlOdRH8dTlS3K_g`B{-v%GOIuKl)hMB z%Q~Sm$_b%hO_Aj#2~_l&L4&5jR=B=)K*%`2#-xM$0ue7JX4p$$h~q)bC;5dT2!b>w zT?0Zh09H{;Rl47GhJ$s)7sRycdEtu!s0(HUuvH9Z1-xDso(_PPVa%X5kV)vHga8z{uY=q?%jxZs6XQ!SRd7b0MDS!1EnFAg*RW`Kt<<;<8o z@j@9*%aj$Oih6!M6?kCxaJ5nzAUa%^2>3)lmbra5DIj!HfCDqOqJw;YmLCxRqLg zO@^T#7Q;Hy!cS02cTDSHv`MjH4-__}4iPK1!DEvt!)}Z=egQ}99dBX*>#sL)>_#I`Q33JvOsvB^6!LytLuz2uI_CIGWavfy0nJcB3 zOGLm+6ldmKq7c)l({4&7RER!kBM`=F))qZgbhM2XSsHBEvh}{4)IAwEaq=7x$k3(# z=rME@^8-jr$yvczD%ONI$L>;~w@kzy_6k{96E~bFpQScZA`UEYq}rpSB+U~vCcbUt zvKw_oFF)8oyAR?byxRHpzGJ#n?0Tn)<$+)JyGY!t1wEl-MFlz0F2UaJar?&`n*UH@K( zn_OZ~G!+RtUQwBC#YoSXytYt@O3X#!f{?fDDb~p*kfcmYmjdHRbAVR;H3OdcvjZuB zo??M(!|2c*oo`!vFjS*UOK&%j%@mg!?}EcQ2Wl#XcMB_T>?ND^c>5fRM<+xgMXi-5t^G64dh1fD7D3XsXin2xvy)}pk ziv;Dd#3cWtgAXS&LUwZ_`z-2h>nOFvg$9S35l(tblEgJXWtN6)Na*Yz9nl)#$PT)E z(P?@25i!dT<@^*L9|G=Rh@@F_SHgwhgf)b4Oh}HoejRtDhoa)slQF+xd+kn(bsbiL z0_FEW2!IupQV7#I0wCw?a?IfZ{D+34MRHC^FY^L+XNw%-W6c-7Zi9p0zw&!_nqvWg z;{VmbJ0rn3#2Is|=@}zm%LnuzB)|@c)`<0QpQwTV^LRhty7=&x@SD3Vyj9xWy?qqO z*Q!=#7erE@d!?32@;~`Rx=I5*E-+y*4*?+A{(8EKv~c(z8+_y%-Uf7)(EIZ|8K>M- zK40JNqqn>U_b%Lgqr)PqPhzv2UmJ_dS=Bc5F`jmYi>1?@25TFg#A(>{jRi(f@F(d= zTzPQ3hq!uNIS&gMNCpL)oI)m6MypE1c&ynG(;1s!(ASjij2-11ptI^2BYUZjOHoc0 zd+yFlM%O@ppAk!A9yJF?Ir)ZaFddcW#`B%_*HorAt*;YxVSJd(0llo zc*oB%hc%i~bSaj}nAm_uVrV7JjT)u5!}Zj#1zw!N&*gqHy;K|`r5`vKVV{<2#eU3^ zGwWgpF1Df(Fl5LHmug(h|Bpv#z=@lWofJG#-2&E>^+cQAM;*KNk}m>D2^MD+d0+Go z8Ap>H=q$!NRdgYpBVcUpdQt##9?e54P@)S?ADJ5pU6Yo5TWEF9xtG`d{fgZ|<*Dx0 zYZo|BaFilWpwtStfFCmC_R`nC9+4w81IDsGg;vz&8(L0Lyz*RNB!?~~aUrN@_mjn_ zhOp+N#{KDvhk~vc1FJDpRnQlz8*XnI=G@v_nt`O?ZUMLb9k=b*3 z;?0xH@q*%T0weMfDJ$-=xEL<%twkq&DlX;dZ?+LE9ch;@R1(Vm7`XZSIn1-= zc+9h>SGoq0#^B|Cvmouvso3PU_}h&5@>c@ZKvxMn4NT&#qwSzX8y!)ibC{1M*^#bi zlVtqc>HMRm3{;O;4sPu;uvZ+fx0|AFFJL=(WHA;uMeJ;P4m13|e4*@|D`? z=uQx2j<5gR?hE1`N3hSx@M~5>?>X;9XbS9mugd`84LUuUd@wg2b!jZ(rsu zW?@GPqEt#TQxGxtxmt7*!$Gec!!+Q^<2Ityk*aM%I)hO8<6ElC!JD1kkAJpcKeQCj z&IfrnK2OQU13pdo?xL#hA4$Wt6yW8BsW~vN# z(IOMxZ?K8?4EX-wPVCs&peucS2o}u27|PVTBZ!G;ID##QJvKz~3>y&*097RZbOixjT}S4w zF0XHemp-iABpMPnXw0LSdjPBaeRJ%{U zdwDYra2;47EukqsGz(4H=C1#(em|#P_v_Qw3qKUQec^dbkH>v!a2U%l?11>!>3TF~ zf~Oa3WOvr!t3cGKy(4uj0Q+A0o9fCxXULIW=UFV=j5N=k%+DS9E#ooRowk;B(McpJ zXXuP@-(nb?4$?1T&_Bac2?!8|S}bnJOfd7&f=kNtUeiaoW7#a%Vw)luF?Z2scSJ22 zT^+Q~)M;ddtb5Elc2A*upfIQFz6$ac{&miPSA6_n?+0TEY{zxx0ScsnUP1*wk)QWR z2@BO-{j0#8bMjds4HdT@$}ZIGuet3Zj!OSwg6eL{Gvr;XmJSW{=4AzODhd_+W_1ax zjdC!;SZ&DHd>L`2)O>fAKinPdr9#xwmJqt8_|={dW5V zmj&~;J`h7JS4T7(vy1RChC?u)Z=!H6$`BX)>_~(2DYBn#fs<(P`sDRWWdvN}ENpr@ zOVed)4x2+XJC~-kbcQg1uWTp32PG%jSu|;rvKX54b?bl@Wi`HENLUYSE%nbMS~=W8 z%dG(!kf3baxLDZ69wu2R#3sdjU928Fp}){3peH`dw|Zl2zb*SD`XndUV_ z?A)|u2B7!EX%qQry5JTF3vSfiSj2!~G!{3B=J^tf^QW5lbnP?FOh+lgAt>`hVM?04 zm&SKXxT(j74A068wA-+0UjD*4B6xRvmhXIj5wq0AgoZ9n+VKag7iPo!v(I{FZ+*Pt z2S(5+CN6pw({`-#6z4ZU7wh>0$a^;7Wx9iFtXcERIy9~cO)et_=wBNpMCuPtv>S-E zUl>wYs94&qLj*+`Yhg6qB{aclT&v6D3*C+C$%R7!#UaX9dvfQr)W!MvZ(w2R)RI~kU1Y9$=}LR&Nr7MR8C2}n7bzAdJca31 zXlaMEn05qZ*?8#zg1>=hzG5-X5UeU9r1n@1IX#3e(8ZDBgz{!0p9C^10}QGCAUdx0 zMQiqcMo^@zAQec!@r3x`qL?r$SE#w@Ckllf;7~2*mCpL z1PnedLJ<)}7(f4IwQVf`xpYsDAu7nf5xgh)iB4pE^4qkacM9(SpL3pY2lq2DOP~71=Ax%#2QoBx!<)hSyMgec| ze2Vw_-^mn5xp5WzO>KBdnN}= zivJb3Yfk!Vj{*9=`8uwK6#!%~ER7Am?s`9e0zSfj0rJN@sn?yfu6m_aZ;1&C8T2PB z%qOK*+~xN-C^`>yO2;FivVT9Yv~mPW)I5l=+W>Q0CCa9B%y0*51(D7BSn8W3)}EAM zB@xCG^{1Vhz9P6f;xM^3H^XVrqlBt2QnxLmd$IJjMv}{F4Y9ZDs#f(J*Q20*=3N4; zgbz|orstMM>Ih+101JjgfZM&j^#!x9A4+5CK@KqRTM@N5qhEdb_Wuo)nO>M0$Aun#K6fD9Y z+54=lXmt63X;W9t?T9vO9;w9J7hDLtX^S!(q`u!$7|slYA)p!%Lhde%sX^COZ8n+V z3Tf4iN=<*EX&0ztK;S%t{$?#eL&}#y9)>KZf*W@CBQJ8L%zxy6EqMKhYY-7CSe`0K&FIs?LuAJvpsSi_J3Qak zt@re>4bA|YPN0n9;&bYvaxX(2d<>1LZg{}}nlSLp)Il>Im_AP*eXhLMaQEw3i9{pS zlCI$|93^+imMUY`z~eE+dcbcI@s@*Zn5t?X{<=v%1>X!I@K91z1dG9vFCBs(# zrr!?#i9yV_wBe1*^zAs1Gyks2R3XBwO?9zIW8O7uR*Y)iDlXO%pYNRBUbWQ5jFGxd zvv=`R3|5L<=UGt)yE2~VKk@k+OD(SUsztBLQe3moL-l)c<)MXLX6>N~21Tj2sv1QC z4|8SVBBm~y(OOot89i09K$^P2%!VY%x~ung(|1D+tNuLuU!84!6~<$!dk)3#7Okaf zdVYWRjVuLO+=|+wl|&kV1u1(Q`y#dTSLo6BcuFayo90vL{yc;Il8WLO*Uh3-{}#eV z-9-7piIRVB2sL1riIFt3aoxjt*Omrm?B&h-VwL}B#`DJW8&NV1c6K}Kxz7h92Y`t9 z{?Ymf*#Fk^cmDQ|Deo;#!STm48&2l>`5h&Nt-_2iW&K5E3_juZ++$9{f`Gw)OEFd^ zmnRc3i?R0pc76~CN0G7vl}GEGMhRLd)IECp4h3{!( zphvnjzk0IQe{`RESj-fs+$a zHMJ9YWFD-BubhSfmj z9|F9;g5^HbB|OE3r?&!;M^ElvRz1c4o@PdHo_3Ime zHD^Bx;E35Bd=+DbQ)u?Bj1QcLNc|YiOXR}x6PCpyEn*5NLh?&#SHcUilVVRv9dmeI z0`>QPJw!Mi7#DX)93>@u3h5UtN@I8olMDD?YFFu9EEUf3Usch z6(_V@=68j3mPaHrS*O&2?b@S-x7dF8pO z!xlfd`jT;1p93FGm9!iOFpvxouHPUkaDXT(pN{8V>OV?H3}4dc#@iNSA)hgpPxU!3jOZ!$ z1ZVh;!=v%f;M$%pZc z}d(P5wm_*Bj@K9jh zR3K-392ou{>GAJ2gRStS#}CJkFcol+XLDWh4pjMR#-q>UcK*8&uB|x9A>;cx8Cmtr zYC9wq%-w6E*W*Mwr{2A?V(P}Luj$wc8N;Qt;_N!+IC_yBu=Db%4)u1K3{Ttqup9KS zv%Tjh0$AAD*#!6}opgu6_^y~%H}L;FC+@8!3<)iwd&S)q5XoXW zHT7^2OvqDW??vw#)6nR?EE7C!UgV4*tIC(Oh-r>yvle7A$8shOY1-2|3K`4TJC|5vM3fJAMF1TMA3T;9ruKq;{I;gU_ER zD(-IWPcQ-O%8Sp^_fqBu?X~nU%cMhbt0aYvpQ?NqqPP9bK!9OaL%x~!H~$CF*41x+ z6#)R=rTwLU?EPmo=1u5<7N&0If?(#VO_BE7?zP}G=^hrLznbP8-%3wBoL}9q-p{9< zor{~L2LRr+?Ij@gL4}Q)ie|+Wi+81&RxVn~IPf3Ca@+(RH_A4>WF`&X9R}`E72Hgq z)zEMc_l`ybOo);|64iB-we@-NsicI0xZF`=Ow(p>E;L#p2=z(dVRT^g5#mZzm$Irq z4mg6uItA;~Jw^4S$IaNgT6}QwsEf*j7uBEEd)f?aPAgUm#GgVSie^*Yvfv#DQ6C&l z$$WB!FvAA%@E8g}>G=qN#o`qD0Oe#|t#1br+E4Cra~qDfASPKy+r~9v$1kmBi2ZUY zS_Nt8Sb>L(@6%dq!5(dJrUj^+$Ot@79v+X(&?@BC@zU%M&$FX-k`Y>K{;rye56xFY zDa9I^>@6SNirj>5*ZLFXbco3)WZE6X<8+bn4YCDQo&YFP@gtvo^YtJ zF=+Sw?3cPqGpy!l*Jb-#W2WU8K(Ymz=dA}&2DzI|kHVt#PH5O02y=LKU+N~i|G(4&hK#^07Jli)M+PNG0Wjn+noaoVRziD`n07`OoH4ylACty*Yt1wH`v7b zMK>EE#@$k*H~y+EN$-q#g|`R6i16CdVLY~P9}g`?5C+w$up@de%wgO0@tt4$JsDbLungGEGu|Mu_M1Jt$n*G2%IC}*Uf1@P?M4ghyQeNH~UEwRa6eC!aN0*?>c zw*a&+0k4^yr2~QI96X!Lo*&8_?z_kEQy;y>obnvxe!YONq<=3k3ia*JX zyUcYDz#a>*Gw{Qvo8tgtCA`kxbHk1R8d~ZFB&h%#>_2GaQFSa-P8v}n7z=vawTa&)^wgeLO?oLWi&DsuQ zZn2ZdC~^YIs{BLRnMoY;rdq{zvWX|_zN{X#m8nlv^ZgGlZCIHpA@Fhxs9MX(m?8f5 zZ&m++>KtxuoB=9S%B(qG5qdHUoKjz02|J~J7bTh|~K+*zH43oJ5`A4`s#A@|-mUmw}Wr zWS(*`-tRtXxbgrMqnsS6%+hyD+bXr!*t731!SC3iG5bFD+Mh3&yYwU=(e<-%eUb3I zq!G$F-Cz}ZiWWYD?{5BkXkQph*_D@(nJc{5{vxQ|Vav0aMg6y>Tw)c4&fhKuuP4=< z?E6Muk~ex7YHF0)m$n{&_k8gp@ba8v`zb(hv?grMIU|+An50a4jJ_o&85AmpDgxhv z3c=y=(`;r#G>S43iCQEyNKP6v$5|rc_>W$CxPy{}X}a(S+MsuP!>7n^2tj5BPYG;Z zh>9wRr5@SL<>vRw9b^$jDH{_KrW#YgB^z}AOvO;g;F=0=2>E^LB8~F6k_dw;j^+I6 znB=~v58Qo#0ElGKa}N^AsOpMhJAysKp&&5_t>GLzqJDu^9lio4txD^ihKS3i#bo z48LGJw<<1($6A2Z=t+z$)q|$y;Ns(UUg18^9OEFa&?gyQ0QyyyDL#Ppeyb&sCS=Yk zI>Ox#O9NbCr6Tf!g2k-C{Y=jLd#Fxp4&sZpFzO3e4l$7a1w0vBgQ#J!a7+;YBrSu? z8b=YkWJm`o>GsotEt2M_`XgdoOCXZW9@(J-i?KR@c=0%3#Ds3HuevoS+mufQa>oG<((< zAh~Si(Hz^2>nv(GnuA$CXB8jHNzo}W($-E0>f5?ZP$RQ==gyxASpal+=U?sdubNPC zo9wqR>O2rKJ|xT~T9j%v9Fv};`#i2O>4Xob|I_i(-B3k^`-r{%ml0l_X?of(#20WF z$Ctg-tE%xEcKvW8){6Eo0atTwPfspYI)?V!+L($ml)u6|ju zC3IPFkd1@hoU!sd8+8f4FdL#nw@>1 zQY;pc_K+fqb_7iJBt-6I_1m*_)5Mf+x__noqp!&oC!JhGB*!C2IpPp!fJ7spoi9(r zh$kpWGv}i&GbEL_PGM$ND2*6pI*TdmfO=0r#t)4)2FGBBSg{f(CCnKQb-(i!7dSp7 zHGVss7Agmdg_agEL3PZ6>erZK9VtDqWf+i~sc#kS_c};AL!WE7`09~*vXU_jjn+(P z?qD>0*AsR!9>efqq40|uLx(t7U>GRLz_J%dDVv%ed2WZB0q+?Lu^w#L{#F#{{Wo4d zAG*5tdy>lyS0hOnGvYh{7YRVprA~W9uHT{V-V`>z$AizOkXl z=0o?u+#F*L))Y*Zg@lbc)g}Zk12#tPG`rw1xw)aLDzc1VUt<7B#@5_H*=2!><@PdX34N;^7wv# ze+iu5y+;>AQ9;=K174r+}yxyM>es2**{E z^PlmlhiQ5x5rpy)LU9m{L?>v0PJSM7ch2s3oF!}o-kdobcbTqZ*uUi4jo}ZJ(a|tF zQm3j70nUqeT|J&UqQJim`ue`5A)yQB!cDRoDjwhjK&qb}o5`9~e0a(>m2O}MVbx>v zVjd#xb%VAfWSz8KFdW&V-jbG%$x3~O2J(c1on5w$qTQ=B)0a$-tO>dja1I?xQmHI9 zs%h4m3|nRzf-d{Lj!to?Aq)J&&z%f`jsMZo^)(SaWIobM)R#jm_RE_CDhStX0&@J? z+of-lbaeRur62lHzsHgTzMw0iBq@I3Jsq~<8jx08?mHUR0 zWDFRyD%bky3pl|8&X_YiY0L26(|$o)?{w+&Hfk5*XJzXNB+u?;gf_N+wPtzOkQbak)+W2YfW z!Nj=#P1R9Yh~T-j$Hk=skEApct;{rUKN&cWno=1%L4eP_kdgel`u(Q~C+Q;ha}u=% zsDzA(q>D@wJ!DBI8khK?g^A*}DrHDItz~1p0%Z6lDb#jI8GG3rXPBMRf1-FseUO{- z*G^0gCwlcO(;Okv+`tmvl-;XkWc;TU=LD2CPjn>+{%mCMC0)9nlXScR9VjS~{hFJr z!<6a9H%3uy2PM?m7SNUa5PDt1!+%OJebeymgj|d)w9|!zGb>n``-f5E$~`JL%dcN| zYyY?ruceDyxnKW4o2ZZma{7wBMX_u@RUG-cOd|f^!@S~m2kO#Te+6kwg9OAxGUSsA zV0II`ExymuN8LX3-9(uRFE1aTGo{PrNpOgytE}D935U5y zSK!F-aX!`z<{s`5&2V07FPD{Q#muWH{AN~6MPQjlSYI0MUIUip{?DJ?JI=!WL9`$d zF1=XDB4UxX|JsvIo*RPXUaj^wj}Bmqq@whUn2X0kX4lJ*cRjGtfZcMLq$I)3glpR#YDmTrY&+j&us7qPex2A5OQ%5& zK86PGI!NZ={alZ+-5=FGFI(>3U4)p1QFnd_{@$3ld^qv;A6xY`RUE!Ia)4P!Y&9?> zMa}zF-V`I1aTjJSPbHi@q?mNX0??BLOl}OJTeEu=Jt#fxiVK^3<0{QA!UgRF$H{To z>F^_;Y?em61jVHAfqXC#dm)|wMxXW4VqX4uw0zAc)`SISVUciUXW@Wlvif?qD-t^P zF&P$D_E80Xj(<3{5rc>=_h^`B5i-z?hz@y8sfAPLl{vKyE)%j+mPRW07=-EDw&Jiv zR7QU*XF<2_P;}Z8$qE;C%qy7C!f9YJa-skih7aZoi%1sf8A%o~1R4I33}?*>r+dkg4+m~l@5TCeSD2M)_LC% z#wV~>BaOghTyXLE<75RfdW1Zzx8vH3n1I3((9@(Pt;2ct@(llS z2+`J6?j(ua^S!1ae+0oYm=(#O&Y;Yw^z>I?Oj0s|x1VW&Mn#6!%A>Y2a3*0ML)ha? z6c#R!^N4(kisU{bRuVrLPa+kcP#sXU5yZm(|RZI2nnAjK-hMYo$3 zQ`H) zPQo3NKe3k`I37udXM=9!KXpr1IPrZiY8y&rz(;)+@ZFE1GUX)H?|!{xS6JEnOate{ z_jxBBckJEG89{HKo%EI#;jBF4(;-f@w+;%7a7HRUCQfxOOU5|06E14f2g*T8Y0wpV zaz`SeB!5LG=cL+S74pK9ih#T=%=+EyW;Xs8SDhqdYfrXy*yL~}$X$WC94a?#fkm&Y z=VkKdjjx9xBYCNhLrDva^#9t_6#6HC(E(k-#R11~&ujP8H`1P2`;;uXqaHW^F$^iB zHx+!H8TjLd-O$m`+AG{Z0f{E`CBTB*3)wbk1}?x8*r#&+wZJpJo9jo@^rQ0Sh6?_< z6@U3Q)^WniMmNTO3)F+UONn~#wMIj$nwHE3oP;E!J%{0c)TO&9p(?9*$kg=z;&y2} zx?mh=7qlbBe(_;1(3(xJvMW^eYbVG{N1vl}Q*fF%F$xY#=bcC((SNC`{tKdVETLLT z&r%L0ostA9C|{&Z$;4C2#Ot>NrOe&ODh1sefnsCCM&sHsDN>;K#zxMHzfAb^EP{$7 zZ4C3IIi9=c!z^g`f^NQ5DQVJ)rPh$E`qLl|jC5^u89vN;>ze(f-WMBQS`Q@hg8iE; zV6mSHk(yLp@c4WsP-r`pp1>%o6)ovOLc`p#6Z;n=2{^wU28KPXyNDkxs&%#QWy&T@ zr+_8<0WHvL>#8hrzb4_d+3Ech*q~sT=65eqW-5z4AlZzHI}tHWX^(0^2XsrRH@ct>rI*rKPh|Bq6sj8FJwx%-YV}e0=24wy681>f!!+?5k4!`JI8~ z%(D<`7vA{fN&NPS^D!g_V9i8D}2xd<_&wA}Gh+CLu=#+W9qPo{?C} zhDRyY^!A~d$2B7SU1@FR%WZ;RVDL4bD~MQWD54x^Cx6dajHbu)!XAfoX3{rZ zZ!w6eARzNsIjtjE=&~LhJpLvIu&C_4ld56jj3X(aH|PApnC#2gdE*TOaD@5 z240yA8KrA#Wm?jyQ_C!_Hakpnx>*W{#5exFvdPcwlZncQ*?XLUSlDu0)UGlFHLqo) zO)fD&ILbui%!kA>UD*@cHLLQmy_KI1p4iw8RPwDi5bh^dJBAz&Un^)>I5Nn946VTj zoMTnNe0tqG*tU-en)%062Uh;C5Y1@_Xh4seRs^t<{+HJ7YL!t17;N!c^<-Twd(N!o z_>uF=NB54vt(P4g!Dei6(=3YZSlc(K+fqFcpn7!CEYW5p*{+|1_C|jF&Wsq&pmX0>UWy$Y zWltCVXb`m1MxWbmQk$p>ZoDoXj^oybSlv4q6RDn8oAF<$uQECc7)EkAVraDsy}fsj za?|8IJM=#h_>FV|!Bq@2fjEpLJpEkM0Svr<}bJ5WLQljYkh&avfUD#{iqW2-V1e}~-r1+CmONSRe5r-ERQHu7)z<)@npY$eI_vRqd;Bz}|ey;;37Gi(AqBPIJ;jn-6dcf)m}if+2fG-`gF zd84)?eC~Zzuw5Z32PKc`b{v-%!%x0k^z73FwlEq}!C3;GMq5r-tb4a0Aa*kWS9;uX z_$n!(uBw9K;`$;2x*AvdzuB2o5z^YP$OESQbYW8rFxs%Bj1jR@{*f(uSy|fuGPqts zgCB@ubaOp&OR*AfH5X!`cRG5hpwxkfgt|i&jVg|x8@zN5UPzjWInr*}cwRd^++ogz zF_)1#coxc}t6?@y_CiQ+kQRB?MEU4*qN`+dMjPC;zn-klY$&Z~{$ET@BrS@QVjC&# z%h%@{k>(eIuQKiXCcT^Q$Nw_sGc)d-yo1(aO~5Z9I~ig!8p zrVj+%?vtwUj6>|K1`*qz?L&VGE5tLd)M;*L7VM=5As27?%0KL5=9&HZN7N92-(+VL zb2XS-cqJw*zK`k~O)E*u}uNJ*N&`G@5ytQ3qSIm6%`@x@nd)@#`mm}R3ycfc9J6)K81 zKjPl$?d%hdptJ%+w{>`zztuRQqo)xI0n}5)7bqFODkZ|i5Do|BnfC_x&_c%Hb5+H& zC#&x`LksJe*_pAlJ7u%)rB~G24!@#rnB8}xh(rskQZ>-)H0fTbi8kyAeWz2Ws+qJ< z&3Jm8GqdpMliI@OP+j;T?5ai*la0Q9yxZV++$yRHBsZ6)IcsMePd}Cqz*w<%8Z}L< zMNxLRi8M^}F}#x2{Fm zRQ08^(9w@uXzU!D1BIVX{i|X9RXU4cX%tCF4?ms?kNp-Cr?Asjd8cupWJt`Af!@w{@n}ej*iuQactmSg}$;!cWG%)RdF$&laxP0=C{_KH}$b8r_ z=G8y>3|mNYTx%~*LCV{)5VT1cw)IgyAQLFhggaK!I3TyGd#}+C5RznI`Dvlo{9O<1 zR9q3aZK^Ju?fKDHe}tL*h!*Vbmdj`;9J>6!^E5dR|3CCJO?-%+PNWK1*Wt#>cue4k zzKfkXE7jE0<6|`y3r})%IZlO2HgQf551YCncy6DrFnWHQd^~;pov2u7_-PQa54bunec;HJ{y?`iY8k1Gjd%!~`+ti}}+Q+S~Zfgd&)@X`ahjmqg=rI`lyENiyoLvD?x9e=@*%7}H zER%e@DwpKdqHw&uOQ*DuhWBTD=zt>f7c>UD@##qW2OI!`wd%gIJ7;pPZ5Fb()0D3>jSqp|)wbZ{QY6zfsYIsQbWI zC8ecx4~BxITHF(`hY;=qAswOKVND^tTHtynrQ7FR0k5sHeeJ1~6HibW3dM##t!G9Q zhP-M|2^7D|8k^yFl2a#zo|TKGyT%e0KG=!TlEwd1HPiaGaSGebXY&S&%al^%`OES2 zQ#;bw(MF=$PbmEvti$LswUEj$99ncTt0RmtRD2_pL}0Mt*couLI0NB$Zy^Wpv5II5 zhT6gsmf4ds$jg1D7DAquIcSM_Aig)3`}5EH^RNXDVv565GojYUm&1kFns4@&wB6L% zeaAw%LYKFz{V(|wikcjOHGDBCN+}$JYhr{>3voyg1CkafviB&=6J~q?c3DXH_|&}a z^eMrC4rz`iA^Phf+zaUg%iXdnhQfxdT3k;+nQX@CBg}g&AnxJzk7RGEGN^$?b`Fdf zzc|;qc4p;XMb{7xvvzqOxz({?M3n=3_+@Yj;+aq?dYwVeqW3-=4g*8%4B-BJzG#iG zNyTbWFjs&>0#mkxRTKgO!g-=4sCt%ys3so|XyxY8VWnbmf+48A%Y&!e@z+808XBj7 zUig!lm`qhjntn;B>R@>_PsOUDdQPfhJ1B?+IXs0Kujmmg7?Kjke^)ir(m{WHN#E3i z{q5$$H%;2U(smEbm;Va_JpB20&)4?%@b?Gi%-3blF+O=gT%V=icGbd&1He|tNZ&-H zayt=lzyr{-yYr_jU1^#bvcJAx5AeXluR&-1gvZsRQ>fa|cIb2Y&dH+a;b%{A%VyQz z9&uOVxf*L2Ga?voYDG&a5Bm9RW_>|y=?@kBv?s<95dGF>C)p+bVM!XizI*<3MSs3< z&Zx%&l0NhY3hjMqz2S1aU_98Z92Zdxb692c*Q~3%HrP0)nmF{H(nY30&{lA@YqsV) z%rH2mfMD=;X^-$XzT^UGXWyzcZGiJ=JgPN|X9da+w3e^|h!qWj6CgEg4YTL`*8gH2 zpBQ7=XymG{^%ZJbg5CQH21W=XEvc$w2eyTbdnCC zdx-7EmTFH>y>XU@KN_y>;N=72$x8FaHV6UfMa|!Co1ou9{m>>HV$Vv9u$F9tJz`$! zbF5$1w)diAOL zfWYybE8*z}^W77v>p#I8&~)z%U^5%7 zTL0>R``ZI407Dn!fJ*Y>BwTb@p&SsB#CU2GWG`m-D$a8H(@tfr$X2LIRX#&MZ8H2b z^(fV^vK9gp5uHriB|A>PeOS)`m@js6<2W*05#GnVd!l>fR%G55!-{ak-$Hgs zVBbCdVtBV46UeB4sGdA<>M zIC=c|kNg*Cv3Ym^KQd%}NA$U%{Dst-w1ZNtFCHkTkyzoNjsg#H!1ZejZ648?-_uan zV8i6QfWPbK{S}Qr_IAS$-Qj;z*3+Y|3-~e*`+v#J6es=vcQ=z=BoWg8kTd;%SJ4~qkZ+fqcI~o(YlW$B-^N;ifkyhrlHt&MNU54Y$(;Ca)yKz!wm$2 z2T$t9@wD!_>Dkn;XWM!J1^1C`%*rD1lcaUR=li+7GVIF4`Asu6!i^(pBx(XXs7!iW zrz1`KDIIrG#OOy6GzOTps$m+rZ%EV*fk&C8c<}H0VdF=U7qk$+lFU$?-;+N>ppBq~ zW1oL?Pq#5E!{$@OE>$cRM5je#gE1(c)|(-J71<>H<;Nh6$uT7gqIt7>EphX;z4Ugx*wb$<=TmfGR;QdG;BOsvzw~|78Sh#X8cSH*8P_q|f$M29d!LqM1}PXQ zWzdUukgCc!fu{WWrD1e9WDb7{hZV9XS@3SI>1sh$zB5jzjj;5sQH2ym0^h?~p*<_{ zeo%?|Vy;sA_j`jlXJCLB*fBT7l2yiSb5%I^yIw`N?$DvBOG&5c2@TqOjgiff32Q3^ z7v_m_?jnV7m$BYVSr@TR$TZ%;xuofqh=h|64(U<&jlgn(Oe!ibpWqWlscx1_ zkm_~?8Ulhn;8IgsX|yag*hP7AKzp=MLRU;-!JJ)GSCAkXuEt5n_~ofqy3fpHMie1N zKdq}r`r~=((Jlc-OauEh_LH)RBe%|B@_2qWS8>d3LN8i}j)!tF%2^(Q?iLx*P2`eR zoSiJTW1K56vEp)z-COM%lll`JgVYnL+O`6TT-LlpF{*IqVB#@1DoA%1&Pxu9Q_Kh{#C)c2qQO;x>0&nVpn-FlHay;G?|5A@azg!QVor;#GA*86Bz zfGX#B`K(PLjZLxPhDjJ3iY30CHT-UL6p`@?Czz0}#WKy6b8#FTSFyY+rs?_ko zxMvwJNv>Ci8LbB+)-OG*?9Ouv%+iin5MrvIXTn@_l6P3x?)<`cZwO2Zq(bvgx)Ky8 zuezjOvRQ1Um(V^ws%=c!UisA>C-`kf2yGi=P#=NMT~DZsG97pafn72*uDdafmG+zp zTh|0(ilUE_=!RIRSyBT>VSiq!&AQKf3|+Li4@`?4{c*@lJws!%O0DbcPFr_;g^YDW z$t)j;I7*2%#|8n0n-}URq}b);&bHD~SOs_C?alkEJ#$bQUB8;pIXx79N&Y}60l5pw z-6#Vl6iBfqp{6%GO{=A3XtZx#O#FAVC6fhp@N|J}dfa(KNMqVil(QC{$uqB{5!K^u zR@P{eyOqW){0NM;M|5K#G$uwA^i*)p3Q>qO-3$d=2bXx0sT8d$G@~86GEpndFTV_T z!9JNX5zn*XWJ2PS zXks-?WW`|au)IvPq83MaOdcRo=!Q%GkKid~-48-E{8=|jFw2HZ`*293a5YeHS-%t( zmmXbPNiD9)76{K~T>U?Sr|jGoK0l8A>rF05TnCH2YA?#@q zy8IkGn}I>~pdXb1 zqd}h1Z~RhhM?z7=dW>!@DN7T0hS4HR!{t*3;9C=lM?;HFE^SPsl9W~MSWZG!z*5pv ze?d3%JVM-R_Lw9#RUSU4Wh^y`n@$wg6B}vbP0nbO4KZRQ&EwMy{k!w8bm%$TUt4sl zUy)+igDx-uv(jtWGvlow1Hc`ctCj{p{A`iF9$BOkQx8ExWEP)wv=;L-RuX<~&C13z z&Yh~%exqf^)-f@Dk^Aex(0yRV)7fVH`$6+TaqM zjR)D9oj9^FnVe<_NBS-2vdWgPAyRhl>LYgN6Y2Y&=CT^n5mgR|Ph7?NRzPYdIM;?~ z2;7zqJl~MAs2W)1J#q&Ky#s8{{^-tXmgq-3g!|~LNWv-++zB<`fOnZ-jek4PFqO2r z^VU!c{l0>n7|GJviyLLf3bOz{6FZ(t#)cBo!A-Tv--2S%iuC?H39T5!x!G;366Ojb z)>nUNY8()yEJXkfJG=b3Lo`^2YAugnsd{FB!)3)zPn@DR*r+x@7a$qN) z7gyvJnZK|5cLQmODhQ9{L5Aw5lxWay78A;yRZv?C;)50V#z2)I1ivV}AY^bpg$#Y< zH*DVxLhjKWK$xf|lGb=BmM}y0ivzK(!E9UNN%QIZWJr+(@Uqv?_c!bn@bM~Gj+A~y zPe@wAHWdkj#+HKqMOC{(1Vxk{-U1c8q4RGLKeoqP4!&TZ&|(%R&VEej@lo*ep{~Wc zFAVhlT(d_y0aXCT57KWUA6ntQrD5KGcNLxQg%KyKr;t_gh0$H&f=ZOc5Qx7Uy z99+l<#4hxdBLo9J;-SCK@uJ4(fDkCTMc80#H$Wp~Wh4-E)mi+HmtsB9wuHqiZ1oQx zG}pT~iId}h0j-xioRSOpa{@7w@urUuo5OeL- zC8aoJK?h1lv=|O1flRc6H)Uce?e-m(a~u-vgnAn7I*W^-6Ndawf+aBTTpgQQ6MOf`>YVq^~aKk!MOL5n89#vWX7Fgt%#RV*AnlH zV;|yJHT)4jcq&dI{*e!VVtH#*b`Y8lo@m$~i&>_ENQ0}V1goE^we3+14J{aU%{blS z(nHH3g$E;-Jd5bIT5>#c`|yK){wDopV_KR!W(Bqi{JZ&`ZUL?n)^82 zbXcESx9guqm5qVYgs(q~4;sw7t>8&#thjh-x0-9pwgmm@Z4TL5D{CnpY*l9%8gNy{ zl7xjc*THP;2Ub$pO~vU6CXxo?KEef;`N$>-aI?ztd$h_g>WN@*l3la|wjGJ`p5I)F z^4;?phYz%Wtf)LbLE_sm+*c_tQa(qTj64v zOJ{K#Kz5*b2G)liq2b|3jIUa27$(@K-qJeEJcqV@xFcC!Q83ZHwN<&f6@A|w@)|n) z8F~e+_(dN|lY)`Z=KV}R_J!+RcNn!|sE-kyPdgq}unSN(Aj5H|9AbM;t(7P#JO)tJ zsu)?g<*scc+?i)?U@12>`MRlA=jA4$aE2f$Dp;PwsJY;Yq5tH*;Rlh$Xuv)X zr-4A%ylu@QwpmSpGk7=`$|5al*TaDLezFmA<4VXyuvygVWz&-sZb6CfH?vokV2C)e zg~Jz=@~vIZepW+I#KjsGIzxts|8*jI1@dq4Io70RrB8XRchQ-%n!}lSGlJy@hXQFe zhX<*urF12pvaVwhzR2>x50e?=ZzA~_43g|h-fHNd!b-B%YKbaWcv=f`@HxUD19gMp z{I4s}R46mjBevEs-~cbS(u!uA?3?H-@vrIuwrQKJkwxT%Zgzq@dh+MJ#lZ#vJ+`1(bx8jUKKG`X!my&x zkYycFGYwX7_viB9f`ruk>p5nI{~V$yv45O>LY@voo!?yJCpb!n`j z0e7|%CKxAhZ!Q9;kpla&9IsDMiZ-Z7U5oBFJofk1c?xH)Mkyx#;>C>5aw z+CqEe70cTc717HrzAlm8^cncrs{&W>JiYRV;5n?Z%t2kr$}+ans!RJ$6!fj2iwZJb zAY~F)z730%fecbm5MM(OO=Oygw0>*C6Qo{vH3^v=Ha2}fyh|c`kZmJ@p1wr-L*WZt z61`h(l)zU;6g%F8M^2)`SNn-zs(%NTDXXD_AoZ&3NDqvtPg5}%7fb<2TcCBRTLKRK z*JXLdAL;@&)=D{VZ-h<|;av+}D_hHfGC$4jM0 z2lgKFh1)hRnZ2?gkQJoa-=6li^%8lT<$VSL-d{Ww-xmYgSq*fg&jGre zxNrQkD~eMM5M;2voH~jQk14G>Fq{>vW!=1S`8i!-JVk#c`h8KbvUoF`A9vT8V|N-i zSkQ>AC(bsu>dx@lNgDZhwhBGg*L&SJQg|eE)+e-zQ%&6r-O&o8!$VwHlX95yn8L?# zyN6pTmij>jG$&4pgQo8vq6-Qs^&u4riSvmdM7XZ7bCj5$<(@1yocR*4H#$U(#3#g(s9QV@_J08@D>BMth6})UQ(#hQIzcOKh8rWcj(Tv6vGA zs`q?rw;^#Zv;L%QTxIjQdcUYR(zDZCgqUL@s1YMKXM~unEUfV8K4_i2k@tFxA+r6# zXJ?RSOX^;%WN8jZ25A-@o4wcTIu)m+h?Z(`+BH*#wbcw0TN78^TG=u^R_gZ4Vt5_{S-wDEk=U)na5*gZ1sO|X7(VK(IQ z-7;Ia@?q_%KRPaDPxi(zpO-rVI|5JmmFgh+pUTVg_J8N~^GC0%zWk$GQMdm*%4~<_ z(`@?-_}_J&0lXbqJ<0$c3*NB$vtGY++moMUaSZJwbB_0263q()NOiF3-96c**PkKj7J5{M zIuAwe@0if#y*?{&u8F$kRm!ZXtXYK$Y1E?M5Q8iQ_;Z|(A3;|jx?k3M&)1T`EOIJq zZ#VmOKxk(E>WH;{Iz8)EvZ#2fiC8-l48W*`^|)~%_@r+p;lX*Hyz}|Uwys=k1>yqu zrV?3uUdvT9O)jJ41}7tzQ!0rGi+oGiL(^yW3{#8vac7IcZx?N&poA+g;pAKzaoM}C z7BUxsPJZJwQYpA^Db_UG#>}qM1lT-W))!q&F~4r)Y5XKx0E$ny?a0q~h61 zV#3kak98T?$7zr2*r`<@zvcc7@*E@V?y8)xU^T{Ydo4zTs}P|Za=-b#(;EGIr;APh zXG59>=+`9sg?564lQ3ewzj5Gm*P0HN|MY=Z!}FAvU)SL@jh2mL&rDij-6i|#+jY%x zzR$ubU=>Fa*uIH{31>-ugn+{!@S_-P^^DpPj;i*WUm$5Rlyj+?b7+ zfbhmYtjFv<((6tBk>sx*?aG>Ud(LkVsM4EP+3QEU zr)NL^ufKizBXS(=N?UxN{svS*@wMlAd5aAk@m!u@VYH}s`>3C zBu;r^^UQb5lyK{L^a~YEt2*=z*ubi1)pn|rCirvCdYSrh*;<#{K|J_e+TQQvw&7AO zdi*jidsoor*5Iu&)7TS($)zVWir2Tuba87C8~TKXCc~dz_gNkHInO}GN9wo!8I_Ps z)$Mo57Av~-E}6e8)^%CcJElF+%o(9$-HoqUeq|qZuPv?ZM$mIa#kkK&;4_RcH5D?0X)w6Z{i@>IB| z9%{VBaWnPvt+w`?-)?_~?rB7;gTd0h-Zmngid-!*A}-1W;E-=sf^ZWXtQI92?N`YZ zJ;J9p1B$IbZY)Y*@w6b}IipoPP4f#Y+Y8*Ngt2G$fCHe{*v7 z;nVl;-k*H>>G;Q!&+Ufr=pGK@`n|Kmu$MwxJ|9gbOay6Kcbh(S-DJa290H2E#9$$@vs2EMvSRAt##4cJD_5-6z` z&K8lqDN4xnU`nn*E|^EnnZ&@>mmiML{&MocrrEgkbZYe=H#hAS#?x7mkY~@HEp6Wk zx~^)6kT=i{fa~x4a5uX&+pmJUyLmUhX$#}6-9?-quf%%0qo>zO%&*hKD`L6l&MS5_ zjpvPuiGu64if)@?tyipCZ8!g34CIlA90s&LaE7Sv2%F67M8jh`1Gi)lkJ6^^00aq?TdwWZm>g%}w)k<1ZAENtC{?8x$ z>v_fhwSr!^z87CSdBgcWuScbBljmO%V3A{%c>{##lRaFzV`;zhIm-_f%N`R{%{Uq${? z;eV}&zU{H>Ekx?A;Mt@jP!H2QX_mfm)Li$pTL_x#9^8$YBf$z2*Aic^-_yJ05V&>; zd4EWmr`>&rzqM-9FN1W0(6x^hw5w&JF+B{~h)S15S7P01~ zsm7PN~>>2;F2N{g;Ab$fS+ot10TiLjFy{yfdpeNh#45f|Bz ze0YPQLYOP)R882`m3uko+RN@X`QBarlWa&1;1dK_$$tkg8t*@z4h~;DdyxO`<8!z7 zA8s1D_A`)HnmVsIXlO5ri;5Kx;oEQW8I@3S4R*649pe{ZI~^G#JNb{P-AR7`ef4-& zl1Gmm`uAcCBIVFjN&yy^d@zZ~B(t*{^xER9jhgee;HnLfUK3;o;BS)+es-LC2W=^HZHRAKv7Bg%B+H!`flCUJnsm9NqMJ{U#9}l0e{<*dM z2P~4-tS9Td_u?w)umY}-{|*L6&G)}Ao<4oZ|8y^(9eGJz)MPhfIoZvM;&NV+aG85B zB%$S33&r&jhI@Nt05akZXpfJej;2#z1Al;|r}$&oCHeJTnJ+{WzBPzcvYl!g$EWY) z?ZT~=>4eB^g>ur}x;U?)?zl{Ty+i&>11_hDxJ0e-RE{>^Uou7Vr+xB>{g!e}*(}F& zAlhaB&`n@z4kYHb(B(4UoV~>oiBP;Tzlg{wrWpx8OqrB?bqTPfU>qi!(krYD-x`J-pT5&pRLy8riR#nyQ(5s? z*Q%5(wycVnu|#m^&}E!U{{6N0f5Y=1O<6WeFY=;d>(8<3{6Bx%%>Q|KbTD{0|M&6P zA*ZyiS%sDDaA(n|FrKFwRAnt`d`T}Dj{;e_mt&DVPWY5%84>3j?K)|Gu}>wO#Hj3(u?59{|c>d2$j^F-x63voL=2(9I504I>9yZSZ!HcI4_y4_oc3Q=~gWy=z1XcGN zsT#tn(H_>m=B(yqUb=Oup&qp^t&E{0G*9dz z)uK(S_;QgG>XZn_>pcpBcPNT-^0B$FrNBy}uPes-7t>xT7+zm-zJ7D4Juuf*jMl6T z8>}{_xqb~YWoZQgdXG8DWl3cmPH4spW%o42$+ZR2P7ET1BxQVrH;IhKQ?V~*9C@c|IC5vmgV97|mX&SA8 zT7MyaSerCd5JycQS1Bb_MW+y%vP3SiWEEI2WF_4hy~K)^jDxj%q>@FCRxDSXm||XA z53ca#EGN5Z#3EVNos8PaUq*XD@WCQnaS$cnshB*|iD7wEGrfs8wvu!{S_e&*fUg(* zrCf1ok3Aw~i6{tmcE~TLB7R}x|LPz*eE#2p;Jl_)Eq+e(*3bLo<7tr)rP7d7`MkgY zF^S+Ar>X${*Q-AcN;?s(pu~dX`1D;A1jn3E zp+?QKdY?RbB9)~1vZXzF0!2sx?URb}sWxw=4StW9&?j=A%sH#jLdJe8SzII4l&qQ* z)eL!14{9n&2%5YISn#M*RULtOP*5nC>qbIlO@$K45gld4u7qTmEp)@WnzK<9{J;P6 ze}nJHlP5nFH5)#85`0JAIm)A~JuBqng&qde%itlqrkqgDi#P?!#JeJ22u0My`gnFG z&B=sb33Q4(RxpH&5nLb+KU}unC}=`GA&X+Jv9~DZl~$^KRI0RM07vXMD9*?mMna2J zLygETyJES(sF+|z^Eu731&K38b0IM3MeYeYN^30RoiU}r{~!jFXhcF8SH&gEAu$iL zGj%9<`8=K?awqc&s_4QPcx3`_f++;shRb?a0-Q^79PKxjuuoVWM|+XLl*W}5pE$W- zHR1IkL{5 zI%zVo<3!#Xcj01G4btCJGQyY%#*t_vPv=euz-EG+6IRnSbGAl%ZOxwR&CzaWT{zx6 z1!f*W$GCLlNDE#q;O!h>^iLr#99vod4kCb%Iz z@geg6J@+qYFwzpoh};mkwsoynC9{qhBsf6nmL=pyiccWPi7_lEw;Bb@WamZg=4YVuj~SFi0}V=!Mu?BoW*# zlPO%#YsJv2AXzV$Mg}~WBY7`^xH;0c4l#jJq8uokB!!~YNsb~+D~XZd)wZ2O^6jfs z@H~}5iYvAYj8d)w?)s(kH!GD3K)x?P2EJD~mZgyuFuj{7MN=HR{nm++#>uuF2FRmf zhLJa?vA$S&q9%R=kYuOUjB}3Xx;Y`_sx^2~Rv%?EVu_kw+JU&@gsw$lIl4Bf7rPGn z=DfWStLh^r9AXKfu5ir~^m^R}I!}?=k<_Q8osDm4-%>6}cv>WqiB~K}qt`-_u#)lA z-41%dj4p&|Vc8cbnat}sDEX-F+-oE5Qb+tEnuKZP7I_$Xg_1Zt~RI z6#m@?Vg7Sag|`6l*F(>pS5PlXX2AD@4U!?GFD)hwi+YMi7uiGeNia&tyGW2N64nNk zxU^L6TJT=71yN~mftHO0eX-eK)~#tV zaZGP$5k!-1 zv=fu`4}nWJfu;bqkx~=uYm}73OiCfu@7k5A5hxo@OhnF6Elt=Y&4tWv8vCT0iqQmp z`)oQjf7 zwb`bT$Y1Qu2E|8c#qcr14H

wSl~L-7wpVWPuO7+p=>h8FVj4l^0XX(dEV;bZ+4|YLknSLeCBl3MwAu&jO#0S~F8}yQ8DEq^$#F*mV@w}=8Ikh7!A9HerLn7N` z9HIrJ7nwI#Y9s`4aa^*6b|=heDegHjD?|z6aQlu?9o7jH145$^3v__4qCxRqs=q5-6+ z5>`c$NP^(WhqN5RK^6j=5NnZdwguO3g=lqi$rd*#4u6O8a17x~_yqep&61c_3G5(^ zYgX|m5eb5G#!SSk3Fh8H%&Qq}54Qpx_z*b&ArOY1EV(H9GX%dse3r-mpVV>Ee*?if z%kn#}`mB2Y^WxbHH~#=dhTI1IDbcJqV8etj@DC%M#2(Wmio^@bPx@K#01<7(QgVb91`b+gX7j;`Nly2 zDsP3ry1@al3a^g?2{4|~pk9=0h*uEWI)f_%=V3JHX2z?}zjf>0lE#*w|AXfT&zkvP z2M2?v59j|rK07)}4EGtX_~tp#FaN*;_qXF8-}fd(HKVlaxRIXGnvJ5NW!q0wY$+b*tvi0BiIRczLEwK z?A$>I0l4p14}mG~RTqI7e=2Imhrtf%iS<_VJjVq?B)dA4`xsnqjCz#w`An+f)^-Y3 zaCHzJ4x&NFh$@1$*GsBJubSu57gbacGGvcsqOs#?!CCHHe422EuFgSdED1VUGt7=Ci@#{ZZ=gIug_PMp&EHlFD6WWRbhGPwU zcAX$+`O2w|lf6K<9+FW1TaN|CBa5xtvCS~LdSPX3LV<(1!xv9q+5D^(PYR<*(0JRZ zZRZ$Mp7y{EZpDm<66G?N^v)K8T%8zK82cOySi=<;!*;@5VZwi3C~mTEVg=#t8LjAy zN1r%^o9VS#97Qjs0M7{$xsyW@lR4VqwB8Lv>|IoJ%=Oz&$d19)ehdNDt%Td^kv0vv z?Mj!pEZ`O<`|?4zZi2keL7+K_kAAI~S7nKB__k#5Yr$*ti?PxMDuLjur$?_~#CH=i z&t*VHD%ridqvT3{IeXtQ!K$q#?PyfZEvy(+!I(-;W7ZLOYRuD+FmOjXFUO3`iUfm9 z(@87V9vPUis>)Ze_Ny!t8jOHfjMsc0ss1|UOg{NaYm(2i463EzIg%35&dS(_Nl{(X zDgkN17_1!w_`6flT}P|pu(CV&MbKQVAao-ZZC0m4ZUeXxbbzgq?;0W^M!?`}WnYkK zpzkSh$?TA~1#=v8>M<@f0>id`ey#@l?>=F1`Dl8H;(CzoOECi}Dx3+&Y=+IcgWoM! zl_u;TbMd?FR28ff*a3to%Xx~t;b>hmCUv|JRJA%XHCXq0M{qpb>jWg(Sv|!z=Z4Bci7U`bF~uw3kx+zZ|v6xG5MT!qWT1dQoeEGjFjx5F^(z%>|G z2D-@WMa&~e6Oat28RKtMEK=Ie!|c{?m4QE1#duj@WQl-EJ{aM8N;ofSP8qtT|2A<_ z7Fd6R2&6y}@fVImLc2+`8uI5L(E6adKkpuiBv8@Zav>ObO)>H6_-d!FqMNjEr_C7>V23u$o>B$<-h_h@SRjyF>H4Dl9`jhK$%G109OGeaaPO| zbtVM&uGlra;H}sd41QW9y$?n22XTK$o3Iqr#`7AYvSo|YBEg0HP+YSsR_%X*s5jJU zLC>Sf70qM8Q1N9qlMmxC=ZP=asTHM{=DZe68*+hY3`R<8qA-?z@Qj)@qae2^!@IRM zCy30%%&8zeq4MV!Xz67*vbg18G7yf${f8-;1PiiK!WhG_km)M%Wwuc4u1YVawP!l9 z3gd9|8pP%KfaXIo2m;XGR;cYNr4p*|tu$Y#xo9%c99xwULf!r4;N>77WG&_;n|at5 zMjeTpcb=PME~tdszQ#2<*aj@i6h-oKEcLUGVPQ3-UmpTN{>Y!z{$G>w`O}nTC98N; zmm9f)SNMOQ9UZpvzaBn&sQ+{?pBQ`3sY^q&OkTWw(!ESn~YF`9xuR9r$3gn}~4eju8RSyo)j zuk8R*nZ_q8gf<>4(}_@c*RmG=TAwn0v;m3vdx$S82^`z|HtFaAVmC=^Rk5%^tXg zXB)Y`dP+TT1T>gJ%fBg`VZ&9e9Kz6@+PN{T);7j(kcZ84Qk)&h)YZj&Vzo!P71Rtlr^tDEgyfwdm8m-quT^#5GaQgYm+?! z-1kA-r}5XA+ZPS!r{pGfxeDvo;@gG=)>WPf}^*@};?iE$D!nYT3%jcrZA2`jqoMSS0gXXa=I}MfJ*gHZ_{myE7ZG%ppKlZ$MfpOA z0uc!fIT*@m*YK2*v2UtJ+n(!@)Y#r75;Iy(UmI^WSgwMp$8H1Io;Rf62@Xjp^VEiG z^+L0rI@-GcSorA^O_GXn{^?V=x7IMVDc~EDQ2zcYP1Xa#%B&l%kBl~EUlUrEVs|{h zwWsagO+o%0`?+$rlr-*Nv_0}BJLdGe%bqy{`F9QPgY`z(VCNjf+-~pesoT4!igU49 zSuZ@pc(4+Yu+=OiyPm$@>6+I{&o=?aONOW!@UVG#s-g6yW_`IU3qlHZ1ksnio!z~V z3`?~LKdC7tY2d_rj@II3c{l>u1%mGyW<>>M7wkIY7+kara=TI3zB@zj<-2MzW>B@B zHAnWX&p!3H&*$fbv(YxZ(?p)&q^+*TO#`=TJ=i+zmP_sonm7HLpAwy>%c!y0Fsv8l zhTGE4skXs%Rv42eU1Z6fcnmMu;w}(25~`6<1xIabP&M9a9^|4!t$!O^;Aj8-`TnW* zV7(If;-9jXep{6O3PpA|@&7eKQFsqBF0FnTy5jx!^B2#X{=esgXAkw??&SmYsBIHV zd?(QoYp*QrzvS@-SWe7a>Se`cgUd8ehQul`8_ZZu6I#<@KnOm3UfbkH2wHETOSiI~ zX(P-H$YG^I) zV|c;p@YgQqfiIJrvZEpIT)&5p3HN%vJCNhQr+Jd*7hjbsi;SJIiI`c@+LBcSgtX7p z#xV0aJnWLxhXcCQ0G_&yF+qrxy`hPYmcz!keR`7`;M{40HU;XgTPhaSaU9k=d>t4O zx`NCP;e9;eI&(J0*%B+~S3x5ojf^H_`Ld%838TS9Hn|t<4RTAF%gt(4BL8zyFfSRz zx0h9Vm1gY14!^XbVRfjb`0m~58@JKvySL_xYMvkSUpT8|pyE7ZL-P1ce0!WP9yhZf zw0^(ib*COQG=TM-6f$P?tLcqan?t{9)te$2qS7 zH;?gKh|yI~dU|A*=0h^jPkCnaw;?%t{`~3lfRG8tJ7!D6Rg5M-=h@<{DBvjzhJ2ZQ z#~jXbupW7gFPyfvpK>pHq(^Dv24!dW(kv}n( zSoKIvdTV@-q|!i~pT9Rrp}Bgb_``<}r=G|NFWd@y@*u5JIjt`}^8a1*zjbp0t;+v1 zI2bhazh?)BhY$MSeSE%-{zuD_3)#A(S34Q87yvIx>ovY@@sdBPmkst&*y1JX@%nFH zfFVR5H1=CrR5h!HB$S~-`j*`Yb@uBA&z0jzq)eR1I=o)gqk;e#dL2*S-*%}-IHba- z9Mak-@wqi3U+U{tB1?r3dJv{lE6Xm<>#k0HQ}^;r+CUpF>Hy9%_?}^Cg&%@4UHN7zKnioX+3lnqF>@S?2ln)0Q2BF@CP^j6 z{rT*}-ZGFyACO3U;!1ek7CqJcb`lb&Jh6G^J7!9_^*s883a3>a`UY%Z)w60l)kzcl zIcL30{kUwcOYI;Yd@gP8cXHcssTMtcnU=jPXme}uR+(w+iNWO36B@ zk2-QedCo2V0k0^&uZo$SlH z0?qqJaZq@B=M~=-9eYVySs+4rD%?{KHQwU5nfm!wTl>v#w?9MoG@{kPVCi0O8<9>$ zu9g@P7v%zQ$hRs%xQPu`ixQ3Yt7M8E;ZvId#nvA;7A3HFT9ELZ(JG#%`GuA31#Z-G zp(;;7aw&Y&jbsHBx@Pd@#0{Nu^zc0+jd_vrWkdGfcV&5&Qh zh{vrlt=8tZ^Phh?dvo&o(e5P86G_?tbD#7eie^F}0X7NSP?8(M1()dwwPDI=LV7tl z*t6!*@NP0qW09+8vDdoMdRjQ@n#-E3X1pf7|8zDDC;d$R?cd}+d{0yP-V?ykHoXv< z{Fc__z&m6EU)>|BvTUmcY@=ogl++7ni^$#-C1iRqCD$Mq%%kQ^Vqojb565SJIr(7I zY+QOewR(`7oAwIh>8wb|vuDqiw(kU8SG7aP8)yf>^>==_n_ZgiS3%v~yc^%Nh4I$z zBF>LjV!hqb(`zN>*XiLEvD|a#6+4>7^G3x)!F5|jw@tCuD^{(xn}06`^2kFD16m(A zLsWN!P3HCTf!1VM>qGbEc$?VMnZ~?8HmITvVTk|U(9C{6Ho66NayporR*UbhE$JVMDHkI5Ruj2ZEE0>_Xy(LWbb=-ez zAVj$}W%?iDstfD71|L5E0-sy?e~~4w8U4F5{`cv#XV04c-{;R?Jota_<8wRzuYHGM z7BbN0m%Tc@kMQ5tBlcB3ME9NipFjB5^NRm#1-)*4FTQxlYYXCSM7(;?@COKZ^+2`l zXu%cR`9}(#{QB5fJ9N_i@;0%PI%rSoCaYs8w-Q!pOa)sBE9@BVE@6c&f%soUHom?5 zw|+&yRr2497tQ*APmdntzx(-o75Ptv|Ft6ew#Tx!5UICo+Ne2i z3$EG#=`}%iAa3i=MwIF3k*-lXd`-fTH1d@R;(5nOnC4eBORapM61RI~(;ktUP^NnE zP`Yli&>|@bU2Z>Yq*IKj4bAW<6Qjy%$%>I#$3H^54PWsP+E$`HP4APxtcK zk(bm(O?ESulijQ+F6Si)m$?T+5?X$>P+T8jxVJ|JAS3R8_V@_uXgc*Z@CQhGia&;3 zl3(AI`9d_|TZ2d?+o`5;eELq_F5GIFPKeA_C@0;mi}M=lj?3iNJLJDK;BuOXOVk=q z;PIMxV zA~`ihX@6ElX`$1YO^QmlbE~XHuyrff_LWl8J$&xrb1VK|@>1OaTj2kLqd|-RA00mU zf9~b;CHVg)e*GO=0qf-F(GQf9(DnQGTONN*KMNyHXqu7m!<0$MSC;@w3dUiwDZRql z@U3CU@##BlMb(T}m8d>FKa~}ab*)OtV#}(C8A}9r4qe8%h0#+USh@hFg$dpQ=_ zkyej@4L{~g664w9fhkK3B>hphga{TtklW3OQdXDAi|M2MG>7jf6 z4_+L+c)0)X<+IZ&?i~cjswSwq-zdjw{_5+Gsu=RGyYNI|*zaGY^>jW4^-a|fR*m+s z_BCfUC-c&+OAYm?b!lY`C82p@7pWF)TE&-(oKUAkI9~5j5WGXVl#`Fmg)IeE5`A4U z*1wqcO2P2@iu3iGL+ydNu41%iZP;M7G0pXBkSR+m2+(`XNiIu*i$$Ve8tRhP(-s~g za!fvY)H?4nPbUMRb#DNe2}kal7aAtWi|BfLpuG@goG zG4;l|E|#q;U`Ijli_laF%~AQxV}_ly+yqxiT2oaBT=bn3Syo(QiFB3=cFTuBkNmBe zlbm_;l>kUomFyOTHifFJLdY94xd~8gyqMQKO_%~59_H+_=pr)jP^#V4b=Jz@x$7rp@KMS0=Y^lp(;9s$dn~=i6yJRf*~vE z&gdmpv}7Et-6NGOdbDD>;=~m5+Inz>FK0Q~O(Pb`vhHNmPX0353xW?8;fjMO`A)^; znNAGLqnhbW#Icp6^U*qJvIKm+=r84pOMC1QDN95_u(LycF%|I(8~;}a(c$y|76j)t zt!nXenzw%5Cm&CXgeaAUoXY0~1}HbPOj5+W9~XJd%9{68yjQ<0lAijbCl#+fRGP+& z<)mV>;)*3mA_65A9LJ~cq98ctgbFokp4I#0$rGt0&6h3h$rC6-3TU5Hj8Caa$dwKP$u3L`9dh7Cf3KZGigpH^h%&p+_8coWQ^bfaroh~^+rJx>IqpCbB(=4 zF|V{z?W0nq6$3b8zd>r5b8PcG(rn1xCdLGn&t7mMut}F`5g3K`(Mo&{0}r z8SjiK1^x#ym_#EI%D5^nSq_PLn4PIZ!OQ3I6p=fbS5QS4#=t8RcoR$^*fw0&yAt4B zlH+KNwhq1g12ur1-?i1*-|K=kcYs8vtbcq(qqGBIi(ze9APp5yui=k6r_J zQ^F=AmJUajnA4z=vrscLeNQ*i5(~M*0>87qiT@;o{|y9OfZf_8+kf+LI5@s z4^`K|L?hfL4%Q&I7Z}#z_qPwy(*b?%pk!5O1CT_ zH&T28NluJmIl0v+SSC9+Vj#-w7<Vfz8ijP&Gem- zWCTQqPhY$m;gVW4&lOPm2e8D*i_aX;8!`&Rkwo>+vg*)MRKJy5wjQeev05V-M;v}} zR=1#!^v78-?$4-T7yY+IJcoxx0^&X`(`c6LxY30^{bi}TV#(oZd!@K)>njw_vSDiC zVlui5ITq?sqSq}!z+q&~rn(?vgUTY|=1gnhLq>2w8Y1Y0@;}m+OOhBjfs_kEL&tu| z5HPVL$9S%bi3uc=!h+!9LhDHqL<-~_a1p3wTaaW=R7Mb9vUru`syFdg8>?W26Y=qP zjE<40`24H8Y#mvoPKS;zQJXIqH^_KJIfq~$YpIVpInjtI;1y?t?8Dr3tg3`?GzUgd zu{>em@s;i_kgg|W)Dc=W0_By}gl0v40SX=5Y#>mh(oPA~;LyBXg0BKq1@-Z#pS|}z)2Q`Fc9VGb< zQLfBBKQ8Us2?_-?hUs2US;efAh!%toxfSBTmbTY0W$Et!m%uk ztbpm=L@Ao$*zLDYlr&DZ?Jz(d4Ks|qIgRzj$`dv58-OG`wPu`iJlD+$Ay=)zi?aGC zn-NRY^wJK*6(@8p3d_;8Nxj&0&^PDpg;-S|G2swP2z7;PmY~<`Hqd#B%#Nf!CGBi{ zOZ%2`LBi7_kxaZ|F&e!ViiDMnr|x#p17>s~L<`HlK*?lY&q2vYb?5GT}`wpv0x6a@T_Qk}ZfzgA255Ea;2Pt~9ZjTdjK2Q%LJlP%=ba)cV3rp$FC^Yi!mbvKEpEWTVp<}wW(s-MP# zBqe$y?!+_~3<08EC0~I%F|}?@i;;u1i;SFX(_+w62%SZlG9lGx3u!Tn8RxWG$n1HL z#8T+tIXRIj@6@oM9gY{a@EU0Y6)-Ls6`Bb>uR{uzTAoj8c5{qs%aT~C6kyzJrb2L6 z2K+Sd!tNIVQYIVif>9?m{fZ!(Y@?l+q<;upx(PG|u#J?OSYM;09A;7qseae4OpQR< zaAG2Ij%sPbCTT8YcGK7=)l`fo=qraqpnMxLOQaUGEw%j)h!E|&7=bw1a<<5eu|}~2 zfvmqy88}=lwxp~eYuf-i*p#mZFmNmVRTGmE`i7lM@?|oHxP7&Ml7#?2lPpeKpyV*r zKamP;Fps)?Zg#jwUZV39BjHq(Y^u#RjYR%pZ#F1CLMw)k8E(kv7)88b>~1Zqmpj55hpLTLo52jam!ryyspaT$V~~+hTJJLC zf?5NTl$sX_xIys>?OwxUj<~7j`JI>2vP5dnp|mGyS0c&MSt`RW9FfiIvNfKSaRr70cC)b4zyN$|6#MWvNsgN51%pb^_+wenq@UG} z{Bmul#-8VRt5p1uQODLC9sLx=wFwo13rd|k&3u$i>gC|&zyalUdx-%0x3Zm-r)wk4 zK*(|%Cg;Y8Yi#9iIx%A8oXEPzF#zoHSih{2N zUYBvQi&sAEs>k#4BRErLX`b*the7rL+0qBQptDTB66X>5zNnBGq(0({U4&fjRflY|D$T!=9>$gI*y18VF z8x)7XLwPuc@FjeLeVt}WOsfQTkj6Eu_>+hP!8v0l;?)FmZz1N@47P_`few6#9DooA lLr<1m6#W^3-yc2?pNG%G=U@2q{{sL3|Nl1BLW}^6M*u;JvU~sl literal 0 HcmV?d00001 diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/Chart.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/Chart.yaml new file mode 100644 index 0000000000..9faec487ba --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/release-name: rancher-monitoring-crd +apiVersion: v2 +description: Installs the CRDs for rancher-monitoring. +name: rancher-monitoring-crd +type: application +version: 104.1.2-rc.1+up57.0.3.up57.0.3 diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/README.md b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/README.md new file mode 100644 index 0000000000..e0b63e0268 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/README.md @@ -0,0 +1,24 @@ +# rancher-monitoring-crd +A Rancher chart that installs the CRDs used by rancher-monitoring. + +## How does this chart work? + +This chart marshalls all of the CRD files placed in the `crd-manifest` directory into a ConfigMap that is installed onto a cluster alongside relevant RBAC (ServiceAccount, ClusterRoleBinding, ClusterRole, and PodSecurityPolicy). + +Once the relevant dependent resourcees are installed / upgraded / rolled back, this chart executes a post-install / post-upgrade / post-rollback Job that: +- Patches any existing versions of the CRDs contained within the `crd-manifest` on the cluster to set `spec.preserveUnknownFields=false`; this step is required since, based on [Kubernetes docs](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#field-pruning) and a [known workaround](https://github.com/kubernetes-sigs/controller-tools/issues/476#issuecomment-691519936), such CRDs cannot be upgraded normally from `apiextensions.k8s.io/v1beta1` to `apiextensions.k8s.io/v1`. +- Runs a `kubectl apply` on the CRDs that are contained within the crd-manifest ConfigMap to upgrade CRDs in the cluster + +On an uninstall, this chart executes a separate post-delete Job that: +- Patches any existing versions of the CRDs contained within `crd-manifest` on the cluster to set `metadata.finalizers=[]` +- Runs a `kubectl delete` on the CRDs that are contained within the crd-manifest ConfigMap to clean up the CRDs from the cluster + +Note: If the relevant CRDs already existed in the cluster at the time of install, this chart will absorb ownership of the lifecycle of those CRDs; therefore, on a `helm uninstall`, those CRDs will also be removed from the cluster alongside this chart. + +## Why can't we just place the CRDs in the templates/ directory of the main chart? + +In Helm today, you cannot declare a CRD and declare a resource of that CRD's kind in templates/ without encountering a failure on render. + +## [Helm 3] Why can't we just place the CRDs in the crds/ directory of the main chart? + +The Helm 3 `crds/` directory only supports the installation of CRDs, but does not support the upgrade and removal of CRDs, unlike what this chart facilitiates. \ No newline at end of file diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/files/crd-manifest.tgz b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/files/crd-manifest.tgz new file mode 100644 index 0000000000000000000000000000000000000000..4beb6626f119e3a8d38294a4f110bdebcd74b8ca GIT binary patch literal 308595 zcmb@tWpEtLwx%m)w3wNhCCg%FW@cuvn3FSQ zQPtg19V@e{EAxHVvnmN9p@9B*eV%t)ecx+oW?Y|nMsp+Am|&?)IZxO2ICWa>x^(;b zGN#>SP2t{|!5$(mE-3Sb7HGUMC+l`CHyB4mk`#FK=iv1DJJHSZ{^79AR{eM85$%Y= zpweBA*ML31YYUGc^V9df_~Y{?l40@_{_XG?L+|}^==Nx2Cbsuw>gIN)_v!R|uYlfV zbOD62py5a6o>9MrBEmNc(i~aHk4uQ%k5BQu`j1mSk}PC;C+%&l9cY>oXYygGEVIYw z5Zy;*I&0_-KmJSAZ68JK$B5t&NP&2QckwKR(;j!no&Cm6!@ z(d0(4(tid>jz`Ej5<5hY@jinQX`@OT-dp8=gvVjin`wwXnTtl;*su{LMwn9VV7I^C z?Yh0a-QLvu48dD;VVj<5uwdm9DsN|H<<;!C$PQ8zkcJmn4r{37aZ`mEjlu%a&VDVwB@X`H?}?v3Qy`+X77g52-U%tnCJ zl%|4psAK0F5GSrKAsgq@J_s$rCon2oD$3rV)b1A3=opiWlQovLzw9N&AnD-Y@bP}6 zR;@>p^d;V2Q?i>ib3iC0ECR)UKPxZN<3kgXfVi2{xua;PN;G;lYa<;Qn(d8RYAz`4 zfaHXzU?@wrjFhThBm#!~Bq;$^B4mO65PUjL9Y(^Dsm38in=Z3TZGnFemDQ({y$zJj z=N46R@u5|yfki@*+aydTK1<1;lZp$uh!L_N7_&Uo-hX*Nmo(nUS)@&HPc*&GP@qjX zi_}_m)_18b7(4sLJcO(s#|*rwZa{atia69m)3Hl9tE@J!!hCx^HjK$*fK#9;tsybP zV5-cyl+VCj7fqQy_D-NAQIeTHMzMi#B|fha`EV>tGGR0#kwp7vJbCvlu6|Z&uHyv5 zpg^Q*yE2ZV_@=I%cLangKOV6#c>jY6OT?mZrC#|;yO)bg^m^&NVbx4e&f{XFVKbOz zzc~}xI_+*!qxK%Dh$ge9_SkAjRN^&JZjKe%K z9j9_K7>S&^@}fFRGt8|%3T~q2a#IeiE$5uuZl16Ye@%p;t$f*On53LLnsm&e+B>$D zO>7@_RpO-g2xR_7wDN+58unJ%pUz!&zo)aHg@59D+7?`ysd8e6Is9G2B-2+*3xrH} zA@zdBGEKXiU3AIG&`0x62c9`P8vJuwFsSIXEr9*fwczdF2A*A2Z2u3O?Ho8XL^ zQAIlq{jU>I#~8AHjp55*u}#2Z^12ecqurT_AWU2o>+cXw_hPd49l}Z##e5yaMZBIi z#Uv)-oImY;8oC;%T`E{D;d)z2yh+S02`GIsW6yRLzsj9imA7OvD#fV@7I5hzu}PQ? zVCjm(cC(rqc+?ToVUZrP5Kna-%hYUIO}4K7cuFB;&Zfp%46OX_a^d0Zls+ZKc;vS7 z9E3HF(|i+9BEeVA`Fy(F5I{JrFzwdOa<^`w;%_@ThxSw|l1owwL#!yBp5W>c|3YD6 zJH|LNc8Y9JI_78gtMjmsS4Fa1aYXnXyS`uEkr0jp#ar*r&KMN?qFM8&e`p6~Q({>O zCIjQtH7Ut)%y|1wn%!5dr>}1}h~r6aH-=u;txt5wW&5N=R8**lx=1nfV^LjVa!O=x z$sxJCxBA6O%5{dqzZL0UkHmnfD!kmbNt~~JKkT>N_Yr$~KWiUxKDpnb{hZ4A*T8pR z%ej=h+e1pJONXo!C->jLc(%aHwW}?`sv-F%!_ncU8uPfQlBMpPBUl7-hz{2rvdkx} zgg0qa6mcNs$ZAI$nZ8zXEQgztYF+FJIV%yAXNI6DLnjbxLP^+vMbJ2dHp1#VGZjZ^ z0t(l)A#v^h_060r`K7AqhmT}b_h~#A&iLse$vgAN=rEme4EnTei!@Y*>PnJnx^5-f z*3~baxH&y?;Y!AnIP(=$qQHI#Vc0PASE#TQH${OaI61lLSW6VauS>-<+C6Ohr;uVT&=?c zR^LQ_S?!3g^mCE-IrGD(>@E)3s=P&F(x7wP1I9X!;^TGd?hQtD(;`4!_O7scrI3fus@Ra z9$9ODqujOIVUCKM-`-IkJ2(|Bkl#&k1O*Lhi^hIPcHh%r`uc@g@-EwfYDg~f-dne^ zw=OX>%^YpUnEohlBDtP!(0L~7>vEvA@c&?-r=Co;)rY1}T6SQrlDca$Xjo`~;33=i7=HxGTM0}wjC_e}e?~)L$0&CQ8G5|^Jaz;bs#^s)bAiJP zB1ucOGX(uIGGD4UZ}!QW`|2)9uKgKDL(8|9Fn@l8%i#NArV^L?sbMma>%dkS^rjJ~ znBXP&G(#IN{obn&dmEWVdKlWvL%`#E^Jpg5%j^DfXZ*S4{;<-R%XbU^ZPpNqM@KSg z7?++bha1(;8@C$!iW=95vy$C6oY_D0&hKRa+~Z~_i-v}bz&;!&EXh7%(57g;`PXg` z?Ah1q8j0h&dV_Ks+<;PFZl90GUc6YJ&qp&pKJS~>y}2V_--q*kyOBbM3p>8o=Mz3% zF9%1j^9@5v(@Y|YtCoyNn#Yn%_{JYo&rtl~_#0Ul>Y|&-j+M@xVS#Uf-o@T27j};h zS(#<8CdI^5%8C^V9HeA4vnnb@Zsm)2uMrY~r`s!adAG1l9MU^n&ml_D^s-gq6;E7d zLo*G;yGPs!=)!NbM`##)a$N5k`WiQ8RJxkLXis+xM5alKD2?2o6*+cxGWV6<=M&g= zpCt&KBF7pe*n6u#D|G zETc9I->$NmfN~e)%Ed9|WE!N?1_q;lPj;=!-krb;4{}OQX9%-3;;!jdfDc{H5#6}; z!l7~t@R3y}&C^58*Vs$+c6#9zCQ^6%lV**A`rD5E1S|WZDlQFMsJe`Goi)JCoX+AJ z+0*I8W2-L$PE=kO&y>1ZKq`Va{(F04=S@~%zTbT{)!10axjCYA6PPYC0e`CviY3v z69lg-n7ab=@DpQ6_KKs^=VNP@f^Gt&8riu)XwX!}oQAOd7)k4R8%#b@kt={{^Lf-7 z)S-xDVXpMW7x>os{qlSR`m2mmrhzInb2T$-F{#v0TfDMY8H(2XA%`HEOfP;+TV7kb zsmIF;%B1QnJDhYH&_+A|udFT^>{#vk*;Y2|}F7Z$&FK38M zkSnpuXtr?&u7(EnhAeVWy9yhNWf?HJ_lf)aU>RoJ3HuG?*uLZ}7TM40cv0J!`NK|g zfOzGER>JioMOIH2slMe{Mzho!2RB-Wk9w9uor=Hb*5!r2I0ETfHw?;1TH#$UwNy7K zJ4-(DtJ-56$DP={OEV4!`pfrANWw@!?(*B16DUrtCnwmt^SPQWh>sCQmBr_GQMKWK zLDWz+WJ+}>gIQ}pI-rCK4iyDrI$%YivVotq7|shMO^ooTVY(nu$dZun1j6bK3YQ>1g!b%`L$9(Bkd!!pdD8u4m({?m_>IQ z*~peFggC+t4O)-)@9D^YP9J-i}cCW0te)B8wJKOAQxyl9j|~ z$5YiXyBE}bnS}nlk-*$6YecROLn%cPwq$4~R-kx%EGGmW&{9&=%$UQ>mYWM2YwvL~ zL{x>M&{Sy4F>bSZFB}wHXToQ7Ab2V*9=D>~y)H<-!OU9P)WJPHEhU_IU2)@8VG;#BibZqtLN%L*Rx{2qXFCeWG3X5f$Rx|w z#qy*zQtCUU5)^CLO^Sk^O5Ru{Y(`CVNzF5oW~~#)ZthmZoMt>Z?)(c`W=c{O9Enw zIOzlKY51VFc5bwNcV_P^OYp2=1Qqm^P9b;45Xl*f!Cz>E$Dq{3bBI|QiYpj6D2s5P z6$;4_CT|}!UKulNVG#wZ z-=w^DKl5~D*!OYc;EcbApxwHIsh_?45t8F@F;kTSh^BEeR}d65por#H9C7JaEF!Ms zY_b;QTqVA1Z<~+i+$Y`^Ii`cXk#3JycTeiV?@%qhRgU#EX*#6=Q|317yuh-RsBk^b zQv2&df6b>d-F%%8G0EvohVJj8b0N7~$Y`4Ud9m$ZLw&!je{4ga2I+-T zK&^jtEBmi~A~UH24uCSg#wM)@G!9wc@%Cyl~V-;dejjSUuT>70t+bWxII&Iy6q1B)2A=yYL11TOD zz)C3|hz%kh_~%b55lH91Tf3mg_cgnM%s6hhv6J#b`&>dNMgrVhkSY`jwj>Y@HSuOsq(29zd zahqcpq>)*;FXe78u>2rF#8WLO>djIV-5@;DgrS~pP7hW$*XvJ~(uYeFR*@T^Yo+bd zA94iHeDCyS+b6^EHD~dcYSpUiL>WsMR69_mR20OctoMor8I^-gm#-ze9!{$q(eH;D zP3s8<@#R>0hO3J&)$mv7K~5Q8{O#+jm{PK6DJz9rgagD&B{?n_5H&MWvBo5bKvija z^0!-!pS$sQD{L<2wHm&$HlODbX(aRBew%i{KW5gW_*s8J=pp}hUqo4(l#_X^kmeqP zCkAGaZ##t)dN>OA)6D&{h2=cUIDO=zw@=hEwb=MqD5FltYn+;04MSC|Gsm0AAwE@` zz@}c!R@X0G!Pe`L0oIvMS@A|I-}e{l1b!Z|rP^BD5bZNzyYwdbn2$;M@vaW(zM`{0 z|B3)qXuXt#W7!07iKtWw=0S-Xq%*q?ntJZvlDx!_6{Ae4mRq)s%BMUwHplHLPSX`D zxEoUz%p9H*t;%UuD>Ng=i*D?q`#h>3rdf4w`$KgmXl-a@?gZS-_v4|QWd5zVp>~{? zqr$N5V{NTkQCb`NQC;O&sp7aO6y|kOmCUeca;6uVn-G+)~k& zj00h+iLn7#f|96pEAVbz*64}__@*ogb)DYvk1vza-AEU1o!s7e0pCH(wh3|Y^R0o< zaFy)|{9q8Q(U;GW{9y3;$ApyoTH7yLVl!=-tk4_==uTZTds@-!zIk3wgW0^k0Ixo{ z1Fb&T^o#v$VK^k$Jf~~2*Tv|n200QLHZ4qRYo4-TfvXc=vRxBz2~>YTxTHQ1{E`)E zV@uQB-l1Asv!y0@^z%Z0Zan!CZJ^| z*#a@STM|sTly5`Kt3ClHE&{hfV17JGvPXVTbKHITS+8OB68Z`lsj8h|_QU0|Y zucVb=I!HzPL@y}ltO|MvKO!TQ^9@B>gx<N*2Y5m{KL#lwKs4A$0oyz-f2L?)zoG z4%^-OxJCR4ds!uVy6i)`6cX}bQ@dJ=J4NA&1M52MZv)E}e-bcUqHQQ(G4aR*#AEAU zu_OO@1$wIgp$9s1BaonEvILRJyB8t`Js2Y4(Ma{(tRy9+Sg;FksWKS3mL%FA z^n>jS(1O2geeoV6RWRg)UdC}SY=Y~cxzeu|i}rNK*yvcoHA8S&3MM!AzyQmW_4~>M zl_v`({~LHQocb81ERaq2Tz3Srqa)tP3KTP}ydmI^MW|}}62{cA3KOv1jzQ>#(02>uxBX1` zKd3&FnM#UghIHD^?c){l_owffvO`Hf+>yBq*SJHE||Y5IzIS314Le(uQyra(A#?y zv>Ybm=7(h+jdT>mxOqXKjDazGHCiurIJ6)Bb%ejpt=NFYN_qb#S=}nG7#b* zz%AtCAlSg-Akg^7ATPF&buBP$6l^%{;S1W%eWE|qWm19<%79Wsc zGk)GnGnBj7dFJxqX+O9y{=G~n;-J$k)a?R|h^>_7Op=?ebiMx=J`oWEHiWRLsD1pxTy_MQ46$bU7b!W@C3;hY)_P0HG6P8UIPA;ZrN&lG?`T z;USe0T5|r*LR!gu`%63arXJE`X(`;f5>FU0qIV4S;KxrV6do9snfkpjLfQpT?A=$~ zbODc?h`asg%h`BAMXbV)#mCa7#Sh4Y>3D$BkhygAygp z2YB`0%mWu;XCYu$^|}1Q=!C6_47tIO>f!UiR`msNs^IVd7NF%R zG?VyoXw3t`yD;6Du6N@&e8{>mSM{rbR`nNwZ0Z2Ir4P_8eH)ObzEcRm&ur@Q;8ijB z-LWahc559o8Y5ijdX!3)^ z9G6pU@PoSah~JJ+0W41uIpqN^ln?@Z^_v zD!&AD^gd~WAq=SMzv}peUcAV=7V4;&M3BvmYfq+OzstcxlnPjO7u}SZ$-9PIF9jJc zvWLhbCqtQKZ94<726Yj?uO_uaM5Xjg)i8vjKTLx>dRYmJjnBcXG0%pb#wonCNwlfgOpV9A z@I^_rZh`@_lV43UEF!D}X>~?^3$ey|1y;NR;;=o{p4 zZUeDxLye1zMUB4*1LuZ;6H=#&uA8?`!SN9$*0Eb}z#=C3y4bt}BcY&Cb zC*!7`Lumi{s0eQR`6na1AVBzW&ZlS!&__VTt~CM=jbM4`KcYt%$i8hbM##wNQ4SKg z2S`I1Is1Xr-o?hx`?ik((*O?cKhggVL%>)B*9V+IJYq_X@SA&#*`N~S)(=g1?WKHw zYOosnX4#>i8`zo$r91aeyxwz6ZcE03X~5w~ffWJ;M=O}&5sie! z2Gw(EHL*1Cbe#wc!bz!ePv&-ZIpc|J+owB&&cXeh@0=Z(-)Dn>^t-KR-E!XnhD?9`F z!K5si)a8ski&$aF^3vCf0jP?3^N~(XZa3CZx(b!*N?Sy84Q^`7@fXN!+l;4-{V)n1Y^x3 zv*6i&8&N@qVP!XpKCma@n&f=I8S3-$)FOFea-pkA|II++eE!G*CaZ*bqH83ly8bxE zLSSYYRvH4N3B7C!yf1*QA;|DU3R!{IP6o&wDbVdF&@S}g{8v9W!2nuuW2q;a*Ku8s zbCsUR5S*-`b z(f9MB@u0ExEcXhVuQWRMC_49cc&6al^XB|)#hG@*-j)63|k<;VNYxij9);1 zpyQvcVy*zp14Hmi@GofWx}n(_n(ed)pY5!JIM!K4ob8+i1$;-z`)T|E07PP+ZNF{+q!x zew)F7sIRR5tE-(r9vB|L0Hu9^l$$00M*BZnT*Pizp)#;QP&kA*0i5ePVdx5uz8;h}n!4%QW2ojG}z9+WXy!g{!=GvV43qdDJ|vK79w z_`|UL%!ZRd>bZ6G%OY(m`oJGk^Jo<(WlHFu?+QfT~~ z4V?ZMC@j zUaY$=l-m0TkkNhI1-_nLl~R~;OugF|l*cI6H~eDNf-~(jstM8{a%qK|i6P}o7rSeO z?SSZP*DlTII&?H3HD{U-kM!XyR2opK$ivZre6KbcBEvfi#-^a2pMFjI2&di`_j}G z;toA^VWpQ~sb7fPlf^p-?`=(RlhdR!l(7z1z7#8$fvPDmAx7;LljaNVWzZs{x3a(F z($G!EXuW4T8mp2(StQKut9r)YH;8ATT-6i=lj5H3M6W~4u34~FnoIJ^VBcbP%*4qw zM{eMnM`i^jMP$J^$ZSGVV2FyvtX$~MO+f9>rOvkIkJz!>Me~q<;rYpea;>k7`IXh5 z#8O|7exQVw9uv2XiI_vgi~j|qr0>jn6{OTwr^To$Ka<YU7j`QGtmrw~>9i)|f{Ozi6%-><3yCh&}i{P+NeHOHCe_*$Nm% zYtPRs|HzC<2Zgw_x=2HW2fAjrRg(R$C#~xUyi^0OU*aXyho59i3@y!EIY#fwyF(EN z2;Iu);f&rMJQ-Q{%JOz$E1S<`)Lr;QTQ39U7b23a-5$5UFh+4@>XZO!@4G(Jr_#ZbkFBU!@?|K1IbgE+3PhmR65e z_WxhPirJ^xtLSOS=gW)FPS?VqfT_G(df7{4?0RkI4w)UK0*hajsur(O>)wzT010U= zDQaRZWcr$00G;S$qp%i;)*r5lD(NK!M$ILWgeeR1Aex0qSC8~P%5G_Crn@UN<{W{| z*%%U#rG+B_x$M?gq9hY~t+_66TpsXT8-`*7(lx{FAtIozCx_czOcLQt!KL#Q)O@xi z!$i4B3c+&)Z?0M6jALG`hcEFiB;V_0rU_aBq$36@K5jfDi)Cy26z3yeZ3OL`e7kC)uAVR z+j%V0GW@Z2x5;W}7F?QR^=VYjh$PX5K)=&KDsnm=g7dlYJ+&6}*=N5YJSe$Ol7hzSS}VKqGAVKVwdns= zt;pbvjz!q19qN_HmCkO=@r<{0NumU(I;~g?I>rwRiIp@uaY1|QpIuiDjnrXMj+xY0V;*gD=Xh6%BSu;)_^+JN!CG<%>fk@(H>QzF9%)Y6rHQXz-0YKjNK(ew%AZX zXt^)?-VCtyDoYcsR_R*amU7Ay6Rm1Q@C+I)&mnhCiHcC2kdwW3+p+$ zS(!fDM$9#0#4t=XgJJqaVhqEaBcG<=Jt%uY2fQclyo+x`I(QO@(fx~02{j8BKJG3L zbRC813@&r3y(<4AsTWPV>Ll@WtaIe=gmA9>(|Pp$rfAO37}1(4v=e0jiZ^2d-0Z9XMm8&WFw zf>0{!fudGiK>`8+1HMDcm-XP}$@8FAgbLp40AYylfKaVd0XU6IDHMI&gCaPFIN*K? zhAsMQ2<`ow02M>gfN}g!_q?$IeF6C!FZi?@$kx~XHITv_ z1wZ9~EM@}QCsGdR1BQSc&4YJSXNH1{(XIHg?FPtq;bQE8Oh4n9JsCMW0Fwa( z;#2@r1;zwio&CTTOB>@21GO{NaB{LJ+L;ay&qeC~u0+e;gDkXtn`Akt*u39@-N7*L z*ah<$hSc(s%pHVTMdkwD9Hh0I+C-XpYI{d|-uVywF~v0#Eyie)8V=j1q?H?j!R6nN zjI2*}yqh*Em(h2fm|rrIS-HBsD(_~jx)Ws?wl0n736xIUziN3Q3+zp({9HOgRD%T3 znbYt$8G`=lQCJ!O6l~@KmTjR@xlc-szGNn6BTw`BffH=>qnNQ%k$w~Xl1UfUJu}j();<=+K-p{=M|Z- zVR{g1ZxYJf9dOEA-~$#Apm8%e!Lf8^Ka%$IVE2g|_2yuuDGR!z10yoW4QCg^0HE7FLcAoNZHM-qhUM?H6i2PhY2S7=I5Pptzz=)DuP;XIXcPPj$ z$I0%XCQXEgg2C3DG8yzxGsmPtq$8}AwU6LfOCR(m)8$Af4+317lmZ&2>atifRt>W! z2WugzLo5|81W`h3r(27zjZcGzB`CvZrZ=+Q5{bEt3=969>d$$ujX=K!P#Q}&b<>v? zvM9(Al{>`I;Y1!K4dlwIkebplYwC?X zONPA_>FAf0LbQfr@Gto*AsYA&sw1JT6c74RK`Y3A(tEza6$JmCHT6!SbyKKDjJFoS zy6n2@=-GV?rf*t2wK0qooNPqh@Z`4B&2qx`*UxOSJ}jI2AwAtgp>6rQwMReSwFaF# z-}VTc8z*XqNAPp7-H;NvQjW~ls9{cRoPF58&5+l42AC7|Vk018XIK!8T?-eH@0{Bx z$wXobX4f8u%>8_H{h}M((f}a6Ci&G>prd|4aN?tWxFDl`Z~!PhE~vEv@7K}*fYMqk z{|2SCG|XvW{|!oy*8r`K{{kwneFtp#ehW)7Pn3#!`PH;d$rl7PP07KIvA{qF_-a6C zKmKrOpg_l10GDnb`NO5xjsA;Ed!GN{(w;#r4gb!iRb9bnRnb?+J#s_-tGCt|{8}3S zIhU@s{?EBI;9AiB8<(cRdDi#RS^)7`y>t)jHT`L!L5{KF~ zXXYbsO$p!-$KxVMc(i!#zZ2xijky zVYe#_GJfXp@D)XJ*4!ej!#qb{!q!m2M?k?!3fz3T{_|`Cz3%z%Bc^}VLw2wKsfVO= zMmyMneP#a@?rcpJhL4&njBmny+#Ze}oPM1T zoui;&>UPq~v(yM0RYF?vZKP|HQS*YNe6=t!%I#$aO!h&wS{QZ4q6viyX;vN#J7NI{ zUPKkB$L`uY@CwIki%v2xABq)q1<0K8=OZgCYWaWXu&k*5A;ka2Ve>~}^76&uXXT3s z^76wV^YTZ@`Gw+b{X&TU3GNK6eHR+}VgKlag8yqnuI(7$vb691lrQLC{yva1Ao7Qr zzKHk&8vHj7`=?v*B7mrm7Jx&rf8(*D-u%ZL7CR#7KjyH1E(QO;$|3*aupz|ph^r4c zOPhwyHvSc#Ohh$pg}jvkyWCE|F@u{(fw3?x{8RA|U^~_>k*`rm~6oAe~4d*~E1JL%6Y6%kUxlWojy-$)WP)Ia&^# zath7CAPnF(sJzRs3p$}4XJlk36C}yV`C}Uh zs|grU3c|l{P6`bZH#V#?z$t@70i6%qud^dudeRP@ zE2|@_bqs41NvE|6F=#UH)U~`4gS(ze^9L6yCFb%+}f39X`Sqy>#@}eAl>w z!U0PEy@*hBzwYG{Z+C*dn(7h@xfZ%dluueB2p$1(mI(sUuPlbKilm}}L@twWBfr7s z1bsdXug9{`BVJIPaT$G^;*8t+&v?$HSZX2*SN^f#URUR_wm$e5Y_9q7*Wn@LabPBrcs) ziXl0dwx0CdO0tu#fUNf&Z_QjdmLve4WGxsU(S%1t+^vHl*crEr``I~Cfv-i~j_gPw zT^_Au%GVXq5xR^(98*<(xRTsqP+;G#PnWaQU}tMP2HX-3p0j0VS`Qe z#;>dCUw_LB!(f#VvPPLk5t=sQ?MYHLwjyH%_KhU?&V>c~n`c|mE7GIsSMPw!U2Ugx zN-V&N4pQqXxO>9rngnoWXK+FxPaxytTImo=+tcpZdWW>?JEiqm&vX)d4$`Ppg^L+OBE3I3qb6=Ux2Y++yI}gL0IojAOJsu_s+|g!{E<{r)&cLUdyp#u+beN zhy$=l`O$wY(yZv;`xJHcPt6Wv70}?n-KVp&X5jyH&n-73gtr^e#6R(b2>$c?G+*vN zzEA%|NBr;isrfO=bDm8pih|a|-R^oZCKe4WY^#vYnBOe(AvaZ=m6-hnT`5M`6|&25 z{5gozNqn3CK*~9`r=v~)lQ#v_7I1|phZ?4#6$xEA_Xp$C;>VSP>R1eXTmBxam0DWr zWPr*yB1l6O0ijg5Cf$1>VX}@wjS`nURx%gG{V3ERqZKN7 z&4uRqv^suF87d6NqfjcX18?0XgujQ!8aud z)qIcE7(7bkLlV>Fb+^5iy2fQ^yjg+a37hZVx)hQltco4N-Am3XY z-(Fs3%6v~n!l7;7-3wuvuJ|p-qg~hNAQ?hII)BptqQj~YGiP+B7{|?B|4$aa()Ly z#iYWtYtf=aPSp}r&dzWG$xJ1Li}(r+QW<@C>OU#Xy?kBj_K3Y@CZRYUOMOnX6&hkW z@R0QV(KMok#F1^O*reUJVw5{G6FcOJQV}ZT78b>2zuwnv6k&0~JP}cej|(-z3FnJv zSZQ=8llYkr@yX3PvfS`(P+}Tgm{6M**LuFe__Kdt&zLgX)44>)p^lr zy;VX;TM@f>27kF|swVuG%V`Z_34J5+g$#dVDBnk)vj--@nfsNkfqS_s0*_4iOW>~S z41~i^Dr$8mVs|P?9<`#f-}%6?vwfe>197j$K_857OP}}ppIb!d0@%GX?!QtFJ$yUq zSXtTpIamV37mn!DhC$8H*}r>u$K=X7kdLwyt@s_b6r#1_t)t2%$J|>-O`pm*7qHbF z%G#U{|5>mXvC~sYqockBkrx~XRvO5m-(RJo1HI9LJLM$wXvt$r%VQQ+GIH^;DQ4%w8L?GzN3lO0|!^!A#V={XId8Hho zOf%>K0uNgBus;j+(V}ryUDwX6?T2`9Ss3Tb>5%W+opxA4&2#%Flu_t#j2*)R6b&CX1!#HAt+BQ>d2BJAmavO`WuT9=z67W5+kEF6=~7 zzt&pRYZ+yW1jLK$-DDW|K6Sf2E6-7k&enW@&qVFFP!Tn!e~JFuT4>2#?S!{#hb(O= zkFJQrx6P+spGQ|9PN9G4lC5@HL}*A`s%kzRE_1UMvKs3!i3*A45Zvd~Wp0}TGpES3hdl-7hpM0OEm0NheA47IIzBh|ky5D?#-rk-n zw{Uwoc|T5XxAOLM;ASC{I4%6Er%=?z1;V+5VL!=aASQR4ZXaC;c72DOp~tfD9yl=rR!H5`Z)~*y$?#KvLDCH zSznCN9{2>>iuSungJ&-)>^_Ce$Z2zY9i<}gz`CmdDP(c9ZC17OT9$ZK68X@my#iz5 zy*fk{#JENZGH*t_VU^oeQGteJo9HEPHzz*drVwKZ% zE@e-}@tkcJ%EEhGw4i!NJt@nu)%Rg27wOiI$n@;!mPE;`C)Yi>+HyKbT~oA?73}+v z@)@jDJGG=Jr0FRF^ZX&N*J3&AX4pbLi>%$+t(9KrO*MFJy=;VBUp<-raZ-yEO2kRV%Bu>Zi5)L*FJ?<0 z);*ppAH~mg*o`E<*AsvTqqy#L!cEAAn{iGXGQqOwA$xkm84Q9o2ya?zb3c5IY;N9m z5RbPLNUOxnMSljG`L-g~dXMwW^3Cn!a=Q$iK5M=|=EZXLFv@by(F$m=2<~=SlVmr~ zQ3+L9<@=bEUbSIXHlW-+l{+9R`b$>5laQ}&b^0BaqvK&r=_O~K{)DCJT_|B0>S|#` z#6(#;ajYU8IvIL?%++i5>Bj$NGweL2N4~n#g?=$bO(a1Av2*{k z2hI92inQVp&8%x0bEjs>dB*reNrZY~e#~65>;ZcBnd8+w8YwfKVBtpmlEA#SJ zl>Dn>PvkqToT{daT>aEUKy!VBo!J$06;4Ri`+s*)mucf$7zk4&8*Aeh8w)g$2$+#f zCUPZ{#-mY)wH1-bc;q9Fa?0_!bjsj5o8RL=M5jQ4(&PmN)Za~2{z_MsYhr<5(9_YW zW$)08x?kZS@_!Mj*92iRXiEEH#eFtB3R6mRxiG4xBSc*xAzN%8E^Jvnd|Zrvw)$w^ z?`Pz!o0rN-u_8q~x{1NxSuAR4IFOQ89$&@RWst+Qk!RN}{X6e$-{KubR#0a2PsV)t z2)a?a!E~8kg}||Dxk-+diF=^l-FtysVGv^<9|H$zJ!9m1U6RigH*K$zS=gK_$~m!| z&EF*NR6{Z*j&!l1xpTqTnAup?E+PTNflL5c&vk=qz*^eqUWxbwk?TBrhKT7bai{g- zR^hT0GZ^yAI7{yAG`&2WE`^o{DFe17dHPUXorrX#=~(g`PD~jZpi$JX>iLA>lPB{LqTP^yM%hpl4tI?eWN{NQzau(GN?bq~O*4KVU zE3b|Y8)mOb&%pK)Ou`MH5p>j=)CsviD)w+3r10mZf^$^Psf^@-|7@uB;P~c`M1Lh^ zWwtF$LO8UlkCu;7Le=73lK}8JB#+$EroN8BOp&b|&&GktV7#F%=ov(0^Z+ttGZ*l4 zRM3kHvy$~xX#yt=3vZ!NZFF^jB9mRf9Zg+Kv?eX*j`8%&({?3^Cao@Nz{iL% zr$e#y^J7>lLXBb`72Ya)?jJLkTfCi$OrhxTYE>IOg43AHuEk*OncQ7slYG2FqNr@RqB^$>SfYsN4++KRLg^e+ zA^dbc-0EJz2TPwhT|}+gy68UJ(KSf!|K-A2I@Z-QpO3FePAOeuggk8(@9H-NrE$*Z z%7#M;2t`dOP_69j2LV3AV*Z;jX%Xl&U=Eht6IIO~X;h(DXj=(8q*l1S(>$uQ2_4=g zhRrHn4drXxfK2ck!NeXq2my=-=~O5W)D`4E=KzFYA;^D%1T5oFLmQZ-OaxU_=Ie+f zGi}%Wm5f(*9+usW1!%dnw28IN*3)LR&VWfMF99z^6<0v47hFq#5&YRb{3@)@_(-8`X5ZeAcgi6;!p{a+0pVK!2|>c({d^7>LXaT% zP%e4-l!T!O55oI6#D_1GUQK704<5(-gOP-BkhWv<)eQYfbogzc##sx{xTbXd-N~*^ z)&Sy$Ih+9J15ZKGpTAPM7u{jeGH3OiR9t#Z&Q$KIC^7M_XI*BPH}eB{*$mw+jy)ue z=XqD5T-*zd@1^B}3aLV$b8}|<-*ht(JqpEbH9E^5qQ{q>>W+GB$});DubAPb5sZiRM7FV`A4rwB5W!xIBPS7kd783 zT_~VJ(XB88s6<6 zUzJcv%sC7Xu+7P;g$(8yk#Et@khqWx?66i{~gY z65H1~j&9@lcV*#;>?dLj>Sf{3<onuEM=HmPbmFqV37w(6$FbA{#ds9{Ol+Kvx2!pUM`U2>JD-E0J-Q z-6%Y_X7TMFXKlk7c2;&R8d1wUP5_#k`yJ`_S;$lWfnweg02S@POLQhzR%7X1yFpVV z2~FI(0&Lb9&oa;R9DB4QJ%e6I`j%~5CaWPEp)tEkK+X?AGI7wg2QBJu%=2z27>(0I zv3l^Vgj3;RCnuQjg8`jOzIkh$7x&}kGXuY-Wu(;voAAhQ!E4KM*2|r$kYf{~U9EP- zSG)RP@I}am>({=HGLg1q2`%#6V#^wo4)_Mva+RK`4x3rjmU{69z3U#cHW?iv|Jox{ z$aDYSZBWwnb7Lj9G2Up=xP-3f654 zgS}}IKibsC>ZsLcV!I#`< zJ9V+b{mV$IiHX2#+!}*sMB5_ms^$?(AA>yBX`WMSBU*c>_!@mY9m6RRM1Oe);#t&G z{W+VW50fq$GOXn5*N5Kk6D|D60j*^sxIiVOs6c}xLL#hU;Q!fZV1;->Ofdg(tU$pE z-JV}Q1C?l5Dc2?Kw6i#LHe9Ov6DanU0SkF{Pjy#~X@hi!EoxWwX+eq{)qCpKj<$@P zm)A@i9$N~`+X0S%%vIdgu&ZN<1A>H>mWfuwr!j5mrtXCL^KYGzU;G=ZDqixmNAkTWSYcA$O$4qYzC)-`jJ}>w+MDr+hnSdb=yhzo*Fp zrtlz3`b!z-J)(wi0!F1_H0Fij+^`rJI;stw8=mBy&6l4N>Z1%??f4aI*uSmZFxHnY z)qyz_v^s&KDg)i~O|NDVomoT+7x#rFj4Ea4S#)%VSq@D{elayazlbRG6d+}eTHe2$ zGM>H9i#R)Y-@u!vQ|og6p5sQJBp2!E#8okGIWT1vVVqSVw`eSm*5$qC9QEQTZgt$D z4_^>9cM*Ag&L0&}?Hgct;Wb+@mi;ZZe&$ z;}k+yg>?achLQJuQ5FUTE{6oLO5PM*iO3Lk0L{h>{NQ8A!Gq&3S6ZOry`+J~5W56@ zR3U>&91wzI@>t|Y6A=qpqxpr1@2A)EsbKsIRY zkvrdlR2?EGWMg}Ti&XkSeeS%aBz9zIyi{>IS{%Bl3hm&)$NG0SM55OoN#MtLZzidq zjjkuS2`4F^jjqNx2`9;)affu&JkRf9=}EZcY- zV(IQqGx@#Sml|hq`D3#~>^Ap-8Q!4UELLps7^iDCczQCq^olaOmS<$TC3!uskQg70 zKA=2oA8fNeH-{oRLW~R9^osI19n>3meoxnJa9(Ay3+uQuwOg?Aj{b4JU7qr_TNE1Y z0}h7c10n_$6dDB*fk0yU9VZMaG-?1(WK#Fzy(|U84cvH%Zr@v^8yT7n##EAc(2%%n zZ9%c5cU-%{ZEoD|UvhN&xN~$_w017sO>m=;TsvHze2E`!Mb}j+ET+G5yVwtsX+g9R z{JODWL#6*r-0iHb&#}UGFaf!?ve_KGE$F`CkM6_AbAuTEC_YRt|ftFymvs==B z;N7wLFdKszRZ><}WV?ercUA&uL&HsU;j^>_1i3Mjv09woxV&>hgjPymQ~cTgaRyW%ba%;qBqS{p0`l>;V(;_mc4 z7h2l}5S0TY`1KpG*FN87D8lU2-khuR?;Nh#pF8}d2+ZN?6U+Zks(>6W+W*G?{>+U5 zMUnqIF@C&IU=fI%zr-eW!2g{oKac{v(})GI(}@4RjcFJH4TY=#!Tl(Q^aGF?DF3lY z>TF9w@;wdluq!SXTz)cnU#?35G>hqLo}6o}xcnsoip2U_Jr#;S+@uTAyuaiGG35Tt zj$NZ}xEeWUhfVqOi)FJ~XYWy->fX62I#lb&D~eZ|oR-{ePj!qary1@GpTu)(clp+u zxsuQ4q)&O`Z>y+`G}lH;2XIxAF}pF*AG;JhYA@VaMA>aO1a78rg$qbo<_f7&dK9Dc zc7)g(k)5b|muW*em{K-kpj zUFmFY-mXyvaXu+JPO*~WZ4s6h8^8#5A4#1qEz4WWpYurMma;?cv<7g7^Gh1aFKf11 z#ZEMtFAVW0LpEn}nOknauTM;eZ50m2%;**Qg;|8dE*?C-)Y{m4)jIclN?Q*>V{yZ~iqv{ou+?DT9Kl6aMGBBf_ zLcrBEfIKeT{>k^v#dX^C)OFKKc1sheC1n_SJkGAyd|vOks5B%iKtE_VPw*UHRLEgc|F=ijyhLHz_QS(b6$S>ocT(kfPa zm4d3$!slr}f|RL4sa9&f2{nT(S;bbUnL6e0XDszset+*T9ld61*w-rP=ROo#f+2UN z8_s)W$m#H;m=$nZ#MdCv$opmzqTh8 zYt{7)2>ZA9^ax3bFQvof5kzXb+LpqZ1mZ^1k z=(Y3k-XvC`xEOYJenZ1%l+{9*yeLoL-hwQ^DX;|o2&w5S&tkG@Lk#~<+8_+v+XKR6 zU*z4%p3onG%=TGYm?-!1rpi}ziw*C4kD*$ z*{((fK-I5z7ps5{QFmt}W2Yf}h2m3hx>EGAtE)v%S|d_%9z!}(V6ELf>M+BzP^I>u zXX?xiLODATYttMya_`JaOP72ti^uG0Fj$s?5TcO01cp(pPt82STnpfbgG3&B5}@L; z@dQVyDp*lc9%rRH!rYGt1NoO&g&eSH|2_dkKv@K82{MX5`R}dnE`hl#?CA^xdrK7U z$HnK(q#V7AyvJ7G4RblAtz=ck|KQ;vvr0LfdmS0Ko95q^!F>AU8wxg$n7EpB6EwrJM7^i$g>0yQL(6srDZl)c|BNDEYAg(J03OAtJ2>19=XuG}i zL9F77ivD5NbG^qHZv|k=8?U)>O&}Pr_^IPhmb9AAsic@*UMCbKRmx%XyZ2i5uFcwH zP8hY7#>!+^YlUKC$wr5+&R|`0%alGvoJQ_RxnRmVZX9jU=MtwnYKj?>VGl=lcoD8O6x;dYULoJ8~iEcI8E z_R{UZ?~LsmkE+49-H|SjS1;pOANSXeC$i9}${aR&;7`_+g?)BvGbg{moSSJ^!t)(B z`H{T8Ab^-cFL1X#4n}V8-jC~lu2BlBg@m zD|B6XlD!vtvBG%y{71t{n&)g9qh9K^$!1ujk<#F88s$dWwB03_1> zp!UVr{N^L=v{*0aN5ZMcpK$oXP3MzhmsmYgFflM;ApyZ{W_>rM=KG!Z4^XyVR-t{E zCIE}@`QHuy-4*?d>#ou3>;11D7Mux7P!EgUu8_OQ7udWE*f;lIP_b|a(SA8xldwV4 zZya5DfvW^wJ;7Q~9E| zjBosLiNMw_cL54{n-H}W%cRr+s%Be$>G_6g0XN4^77|O5rN$4(-Lbf2N+$f6T)xO| ztGtsa<6m#37hxC_q7wCqsN5pHUSu;d;;qa7Rm42V%@z3nF1G32%T(hh`jzsh6x9g(M`$u3y=yK&kQG|k`Ftp z9*Du}a`*P|amC=4TW~^BHgG}2U%T`9`ccimTudE=qO2$aBQ4nsBrR#FhpO9bpS{r4 z#plcF<@`imhh@OrXQT~8C*GKL6Bx+|$(%mOH7xGlzYu+nu;J^bEH3Iul<6uQ(b-nn zgO3)u9+6yr)b- zx2IzUPQSOu>d6VWuvOMJCI7k@;%ib}o@T=rR1KUOrH77u&ejYzL2_$O?MD-8x`3k~#NY@S|+mk;eBziS`TqNnZ8QZYlzNhw8W zTuxZ8XG}E#;!WAuSMX_mU!_?_KU6*V?fYOrcu-nYnM%)*G(&jwqho{?C>(|Q$kSU*POqw2BEtwOCriAmvxb&AFDPg44w6LhGUeMXQ^~h_+!&&Ylip8AHZpb z;u#)FhgH6Ji?L#pLvYmGhB#$WLL<0bb!Mpnc*n=_(fy9#H{wI{9+{-Q@ z^1l&9(A;d?zIMDkQx>eqLMT=i7C zn@1=7^+e)hE)|t!q*IQ*eeO|A+47cJ2NwIZbXDFiDc$xbX`^!0JzA3J7z;dIO&1;? zOuEX;-^wn=$1-Oo-_37IpI5P*9T$4e0!=f2;qp|TsMz@*eoKk;J{OBI0&z@;HAGg3u(tGN3 z(aY=Oc$ivFu^1h5wk!@bwSrO#EB!anY}v`97O>TNF?^@xnln+==a;yh^zPQbM#MTi zjl5f5adTb+LgA7J1wlPSS>H|%&se57H@X&dr^pSE|iMQ8krqhT*m-s#5y1>DEH(L&9w~yIU+WV*s-jDdFqnVH6tKUu`5;T^ z&4j+MTz*WPewBlD4P?Wwv~3izX!!2r$`s=6U)N+Udw7SDt6wf#T&7WBGk`f6dlUaFmO5FtZ$2b7;voGph1IQtG+myvy)<7& zuGl!4OfyMCZ6W=}WjP={<$sg(RTY|RtbH(tOpdvs(a5Zo@n$1g_p8Z7t+iZU+Va71 zN{rGZ1~W+w@5m#9B5PMC7`>yoy>FjRp?L;%-caHxvT7MYKzY6ef!+nIm33|C1b5g61H+Yv>!^{2G9(^f+nA9||9la$)7a{~x_;~C z(fjqdisR}rrNj#pO?DaP8_PGAdud?sjWSv4c5UHCfzjW~`|WtCQSWd>nc6YGi7K(4 zEb|Z_uk5oWFAJR_6WHowLdeLh=KHJVItD47=n((QAxlP4V}G7A zK1VcnX@q<@zj{2$-KEAj=91miy3{PANz9)f=> zzuH6D+swm!q(}`vfcQ+D+eK0OaS#6fC6}y++DEFbHZOKmJ#Xj4rfxRnGdba11>{_r z<%u0s8r|_9hokDWYfue${xQ&4cPCmeZr_sm78)>`kB?ZL%Cp7%G}iE+Llql3JyJL^ zuie(m^>_X{ccNVDeuMFx1Yg1G`PoHAR;FfAZRwg!@VHld8!PKumMr@%jRZFU0nlnt zErp)EqkztyDcKEMjfP@;Wq!&^7b<+bqLGlsb#H4(bOcH9(q-t31V!>MS?pc*3OODf z^22yt_WRfLfOH?_pY^vn6EV2|a$^xb)OLweMRwraem%LV5kY!d=aXJbFTv79cqEFf zVJXW$`Sn!nk+_mZJcT0cEllra^n#Eh5eqcTE!U}yPzqYK+k5|Ywm%X%9SqOAL^>sh z>&A4nXFB2sKY&#k6K?vK{7@+=Nk_|^!%95Ec<%p$2G-!iiH8(kYT!qc0k37iO95jBGU}Q_l%=?w)a?(o zVr~1~ilsS6G>7Dj>#mo3DG#Zwp_v=vVDz5!!miqfLD$&PzK^p#f!dBhL!}0;7;&di zDLFSBh$&m?`kTCNt#&|8hJH+90}S`^(^=MG6eHr6pcFS{I}5nD2uqClVcnMy)&84j zP>0wdgtTTr<;XVr5L2a>FK1e97t*0B-X zuu(o7Og$z(bulcB<}8QkxQ9Qc1615ubS3nUh|VDTEyqrTh0kn$Q~QUY`q(^K@{#7c zGbY#Pq3_riq4)Oy-e23x=s|t(;v#|sW_R*e0F6>H0cI@OMwKRtOj7nq(iN()d0i)0 z7B~ecT=`VxbLThg77#=uFQ~z2G2__2YPRNPIspLJ_tmu(WN9U*J-c)7wMWpe3_P!f zP~e^ILP)1naB@%Z%i(Ewjq*8mW_ns)q4#?tvsM7m{S}|<`=-~FiTklqx^T00LC6}B zmF>%%)5l=LKRp_fGiDa55?gjsbr6H|BmQEx_Jl$tZz7+~ z4PB%2`u+?0rq+EVJy9FAx4Dko}I_z(w=H4GE^2SesBNba3`Tks(zf zXqPsU8?H0N8`mG8wY>28ra3^>@nP+;dsk-_Y6fhBCGFG&l-KZt?WtLrws@Ix!|oz5 z!)|IM%~dF3_sWA=mc>?GU^3io-wt-h7JvaL^hJA<#8!G7V{3)Z%j|>GyNPhZ*SaO& zRU1qX(8k~GWEC8=T7T95O?cY+m=j*4*yw)m-m>}??N3BF#+84tM{~NkiaYzTj`P@vwuD`eA5!HLva!OA9Rw5$D(QpMw61Hm67ZR zTh%Y{q_sqme#S};rdWDh*X%OazNd3)I%*mX`@>~QMZMZ_NWXqXE;7%YBQ@9x|&+wv9HXo-)0DaLcrcVnqXxtCq4G z_;W?8K&v?C!X@Q1>p}}tN?e|Hr`j}I#n}oY^1V!W+Gry+Tibjboai1Hh#JW`rcIEv z#-OTlD%#~Z@}p$69k0lIv#C&69z6XG_6Obo49kbvf;jjg8B6FixOWwIquq+pT^ca8 zJik_qWI8)E%aG&bDSB#qNn%b{@OR7>cOR1L#>Y?p=?{VQbjHfk-V47#!J+FTz+9NG z$#Bt-K&@cIfHRCLt7QQ92U%8FlI0I$h#NVEd3`MPB?nfv#YA!#98V0lAl}sVU?61E zRR@%_E_pRja%FSi*)Ug@5SN;ro8QyRsl|MEjAakCEH$cXEF|+4TG2=4SBZ#vRua+) zAJ%#M4xEIQ_MTbaNPQM6d4ydVB56pkh@|TR$-!iSpVz;$!z?Dc)WSq!yyvKa$PVg*vw_{g1xQfqPA zZgKOrIHM=T<U*8=IpFt~QlY8^x#H*J2OdxJ|FvNv_; zzsB<6gsM%alsPhg{Y&k_o9ygPhFJjK(9y-xRh`IT<<5j(Y#>Y3NUJHaI}Bu~sE)N( zlTD;Nc2<~A^Wi0`zNjT0?)bNs)$lBQ+ydRhoXu=}z0!PTBnUQAwM$>l}ZS`%f0hCsJ~9E@Fogu~zf7D1SPG< z#^BaVPt^4bMsGIqJ1DuRy11?mp$!02u?hzuhK{iHccBUY$Al{^qt%f_5(f+PH?DPX z)WuKwU!d;*My?srlSlWIJ4!;6O7*ve-wdTEM(L&&(9#%)$@L&}bMPgkG;+iwbd0(D za9CYxx=eiPy&s68HIJ+{i=;KtQ_Y>$J-$Wf+?MF^n6~%VRIB7X_Uopa@GyCiV6&wm*xDsyS1gIPod8R$Z|T zmKKvHiTyTL(L;qMu^6{5ROiUE&pLy@1${r@6&)g@(G#f(`tnrpo8PGd~s#-R~>v@)shi3@dq;!|>bAFI8;rq99UFMXYhj5$$z&1&D<{Xh&( z_a^FoD=~Kc+sIN81n80?3oL`C5YbCK0vJnF#XN^V3iORII9-`-%~u;KPw?Y8b%S*x zpg(BMoT#BAr}}^$Ue6PNdSQir3C>*omeAuwK>Oz(Db$`PdCiX_D!I!h{r#LNuPVFM z=9JeZS^1jV5ag4fZYfF+!j6lSBzkA5vl@nOcBxRk0I-B9wj~T}^Ji4q^3q*%3+u&| zfZS$yUQSYTT&yKeg7rfZ#ZRRubtPz>!rGUA2P1ZzhoDX4ybU@5FQ zI)-*_4Vp9sC;7b}lw7Y)3whqkVgxD*W6%FY`)K_&(BCGIZ{ItUpSx3Xb;3VV`dd!q@A94)b zG*%pPwpho(kzx6M;WYz-G<1KD>eb8-lcO|e8Rc8w{1CC~-2xoT7>*l`(R5ibOG+^H z(=6j3;b4^DTyQS+qiM#?uLYm!?o5`go3x{a1d*0C0j4p^^5eA2_v4p`sFJD2&XN~A zW@@vF8O}j-ZCJ`pFHf^5x)vMVL&9Zv6>BZA8O63i@?p`nKTSXc?*rR=QbtX_WLYCA!Vl1new3SwCY~Tv}d@bQxX37TCn1oCp zNe>wCLa=~MF_J_yVcu{l)PF7V4}&cCU2z#Vgy;9=}u0I6Mf^foMd~xnIGPd z!U5bpe=pyi0%sgMdXblT>18%&R??`s`9fpwzP^SndxpJ_kNM&h?wCI!AfL(Q*}`OR zi3!I2Uo6rC8ht|BzXqUt1^|MFF|oIGU;UTm)5v!}f{zPy9*I@%iJ!m8fT77d#byiC zJD~$9J=2D-g_k?{dUsTF3E%D$26N7D-j~Mg0S;4^kCAFF-q~(mS49aM&i46w>k^Q) zmLKXaPa+q~&f*f@`GJ*S-sw7Pi6&;r0^wtEcmQtk(fh^W7wv82_C?t^6^2c(Ebsvr zbH{Ld<<8=%%$Zznk>09yAUHf}{1F)A2>wYp;|RKq3j0Xmd)XfRW6#uvf7}m$Z~SPa zjRx7>C7{%iJc5iwCy`k8PAx2GY?vxxehfMy$glV=AVP7?_&XTg{tb(fruGB~H=RtVIH>;^`~XsU86 z)=X}#{n~Yxv|04v`SFnjV2wVO=^CJ=R%bGZBs3$fP8!ofBa1osHZBDj5c;kMC zn6J^JtDR<@=n&;JFlSOc_VPXX=RUGWEUXV0S3^JCt;O}6kZJ1aAP-g!RVs+H-nj=M z9+s^`2cH{Yk=jgrfp-auKxw!sJv7jICXuw~R!3u_z2ulHj}rw+Dn|#{tBAWa+p$w0 zWpgzy6V^bH!#%*4gnvlAIje{o!A zWfqzaMPK}YZr%F^zYnFGW%+%-E@N}QB}5VYl1P{cR3WbKf#~J8y7u2Pg3t^xBLAqN zUynIXNZS#o)##u`^1Ebhm(xG3j^;Jsx68HDz#{vpY=zY$=#<}eBRvzKmmi80jn6$~g5QCt?9MI+l?cHtKR!d_tPa}O zH0?CDI;BqBY3(R2CyK|8`r z)oCjhCfYT+AYzX&9p~eAC2giNDOil?*48Jxa`A=CErvg+alBXyd3#z~YA1$J!>iFR z@ukkE`Zye%s1yx(sG2)TI=$FJJV<>QtSq|JjEQmqRnxXxb$bArxY(X#bb1P_Z^Y%X z!L-1le+7;|K_ERo6BkY77p^w^nf=m9^LXSGZ^)^BL&b!SXA*$_ z*Zi^`WXE?rJK*QrpO0;O=yv;WuloO8T7Fu72E4FJ0KB43?J?{ehl+1nNf|!FWp=E8 zVnv@%Lz$!uzGeUG_y62O;8ylf-8yam+u%Rj9u8RQ`|XrNivy)*bt zxEM5Avg@ka$TI*($QDWNwqd}1R4!}Uu5d(Rjyl95==n1(h5MKd*f>yQ8H#Y~G}34^ zbaB8gc=PO4Og(!Zzw$oglj0NfLUP{m&)hCJ0^i;T=$>3R&}!ibAgIwdMi#w3idP@a zyWByjB2{!|#&0X@rBB5zjcwJHH(ilzkR_gYO*-((j}K)#k;?xyGkmkUVzlphb9j7{@OwJ*p0l&N)z6jhHLvxbBy)0m8onvq^7`Fe<*(uo zT8ZArH;KJH8=FTx7i|+F&cp{ejY!;M@cv!#L9)U=#;=Lt?Yvpq8}Do780F2obpG(X zRUVTWbiCW!j~IFcM3-`11)RwAulJTt-0#Rp-f+D z+`7JNB#H({@dw58>&lsf$kSPq$NCQ8^xKls+gKD=(klaGv&Vj_+U$xsecp9BAs6MT z%+I^RdOQ*0saP3Hie;Z*`qZjf#{K;ZBzhM=Ylb=-FAZ-|7%HM%Dqi`@NH&`#bv9K! z8^q2Iqr+^b{oCEa!lr<5+5o+r!HdofU_SEh!K55*PCI+y*3NBQ0jaT6nP2z=JHa*_ zXuW(jvhMs1N|I2kP(iZ9kRzz1hnP*C38csRs$TRP$?R2ERXOv!yiA}P(<)2;CP_6c za2jZESF#CvukD3^8y%dv&uD7{M?U6_!o6wWMd`X&uTl+XrwFE#`!Ce9_QxQ2JnHG@ zRzbmdXMJT$RiR>5O8viZ- z^1sjc!X(yFVt-zVo@sy?utbN_QNaR(jc`)I+Gxr~cnJxMuK*6GwjbUT_LjrJ2voWO zg_LAZvkhVy6`9xK=8f7ZTi{=da_}D!zKe(~$^DP6OSd|D_2}N!FxmlN@i#ObSouJe z^P|$~GE-1;j>22*#{D%-1ptHdgmq-~(|h)RI&!#;Ak+T1#ju<9#HOadcnmu?{MHop ziN7gUw{|8|(X`ZEk|8p68kgfr;IH;-p3y}Mv!6|mQ7YBpB1Nj7ly5-RC9uCCE zW@dIrd%N1+HXi-;zpwU}GACLOs~0De*B|D^_pUmb(?{8Eu+n7MAv@6x9>l0`R2P#2 zvWDoO{+O^(S;*y^7>V?640ePu4h*ZNC`^biuNp%7LCMC?DfO&VY=Zwy_w|Tt>ec!e(0-#cH?zAuM}(`@z}-F$V)(cN z2YwlfeBUvPrmt2oY7Yx`ZD`oBT3SIvyO;DDUTm?BZgb`?(9O@^92fwRqf#Km3e00qNBC8V}6g7 z32t}56IfL(gMPZN5t*W5c(7ed9Je{6nAv!1WZ=Knm^8JC`hbt$|K4P27 zR^_yy^zM2F{P{)9Z$t!guCeM7RGA0Kz7m*cgUUXPMle?xjSI%BKs7T6Pqp@eb2@07 zW84m-V5{C=&1AA=E%mVI>uENaz1=?O3*Fm3+R8P1TJ{I>zh zI-yv^>fJTt>(FSIRzkjXmlz1QjQN&Zw<)FO5WhS9%DFUxhzows)O!EYf zc_jfSpKb=x?gw9+FffHwMMt6}V?H$sb^v&K8h;Jvg8c{~nT#TTarq@i&FP)Z7RYo# zK9=*0r8K?F3j-8u4py;BFH`inIfZ7o&D8B615YL?hFR5+CJ81{DhHO>a*6hkR#hUl z*PDty=z+#b^=n^)ox;OmJS{3?l@OI`uD67_Iil0uUiP@*Iu_b)Qhz3`Axbq8J!ErH zBbu$^xQMr}XPtYN$$m)mmicyCfOs8D*g}<5#WmjIn^L6&Sia6Xy^Zm~$1b$ulX!bc z_Nje>M1zE`OeWqU1%hNk5fiZN?}_dbFJE~ab;F?o{&>!^h3r1F7DBms z*UmM$4A2pGYIq@aO%m9{NWRyRDPNM1CQG4%t4>7X+w4gCf)OOX`#+Aalw2gej+K1nH)9uha7KSBqp~n`1Xh;;s-KTAL)9 zWam8X-UdDBb?|b%=HlQtf~jPs7O)fS_TDWz1m8n6v6aq|R`#q4PbWs;emQIF zuG~EwnlN@>XI}jIy2!g)<-&yMMb)-zQ>I=Gv2?eoPx%*`B`$L?iKbs(>ee_j2{j*4 z-(JR72F<3M?`whJ(r{b&JERW>#nP_+5$$c^w=3zSGT~*(ntp|gjjdDrwl=sc<&2%v zZjf_m!_{8GpMpF!CU*Mv)0HjnB9m}r)8DKG-dN39?|oX`W6(vH^+gIIpkrcc>FI?) zmvR>@0k!RweEX=HB=gIXn*EXqZ1MKlZeyqTWZE(sYbJw&8G7>h9iY7E%98_USAGJS zi86b?e(l?&Q(E4#&#BszvM9*_aO72)>v&fz(;;{e=^npkctiVWy(gg@+1`xkWLg7t= zJ%%b8_uQ;@CL9rfc)B2Q%oHk+BZa$)XcyB+wpm>GCqWzZHv1Euw(le|U@NB9N@aJ( zNF3^G=!_iHeq*kNL8f=Wd1~a)H6l|Nio(+PbePd+j}3#o&;u=>ZZ_rPIaK0tAto{^rv)B24{hwCJf8Qp$oQ4euLi(X`d zTlSiVP2b8|T+SiFJ!nyU_lYo@vae}+;@3_5mxlM}7{lgQU&Ho3|A}dunt!(&Cuis3 zS6&M7{zR>Q+JE*D>TBHUYirj3>tF9y)!ORZ3wqZM=YEX$tN*2` z{mCG@Zm-bwGw{Dz{x|mjdj_2mb(A9~oN`P}q!|kQ?hMyxut=R4C18d_?&*M3KW+GZ z^%j>V5N1PmV2$|u3AurT4AW_hT~^o#URba9ef}oNacbv?DPr0lZV<}Xi*iIah%}K8 zEzR(xqpT>h)TS?jYFQQPIEk?a*sjGu7D7qUhr7a7?Y?n~eu-u|%rF7RGoN2=~NS^y^1wHztBfarqXr;wOHTZv!p zEYfW3S9fc-ZFu(AkcND^{|MTZt!7X4TPC?714!bzGy zNWrTx^|H`+wf5c!vLTM!?+!~bVOL5qkry2O1azSCs^6bCGhYw6CiU#C7gS=TY*yu# zN0jwOMyAV9n(&9Q*Tpd-2tsGEFJOG%A7*oy_&F(;PnLCdjnXy8pv}40*>oNTe~wQg z-P;34wYLp>S_AM?Vvwp#x701zzveG(((2XgKe@8|@6j@qB)r1jHR-Z13Ys)stm)bK zY_ia~eV=5s)nK(Zed2aTlf5YQIHPUed~3`0&M(wAhm?=pw<-?Dy=?JSJuO6-k5~0b zlms>j+1^KaT~(3H-u}=izHB*C*l^bUJf0-?^PTW64QVY-Pym##f=410jBg2$hO4b6 zJ|T^x)f^l`xW4+ly~1M;YyKyRP~f9A9`|p1fqIp8j&X4C%A))(3QzqeyomBs;=Q|9ZbgbAqW5epe6=-$dU&e+ zDZU$L#SSoh;|{UgtAQGQyu&38A(qB-Lgo|B7jD2>qLX4gL*QU6@(mWNgo6m8Vh=Oq z;yiACc#=|Kky-Tqc-_AngOGeCZKJg8vFp9Tv2yXXYNf+`stTvtVy@uW*Z?;x! zV3&Rj_5sNqCmi-6TA(s%SzTP*qn;!E91SIvCb|LH%r`wap^G?sh<=dw?nZsh#r@+9 zz(7SsLVvR|B{h~PwJLwd@ph0m#aC0ESP07dzAtg~BTayN3iy9r zrW#bwX5ufuEjOz3aYCM@P8{*(+uJ)FK$Ls6%RE8*vq%$NNV}3Bf1-(!kg7q{mzw{= zZek71M7UJD+7BotjODgFzP^P7a55Xbe*uv~g?$`Ca}QH0Fs;%62hRZPct2JMFGNLZ zvklraaC8aC(^dgw!)afiYyz0y!^;mT!dzIj7Ku^C^9g{1)zN~b;0T><$$CQ()ZhIK zG#JaJpPy&$m^c52y?cn#tO>V8oi;0N+qP}nwrv}gwr$(?m$oWxRoZ!L-#zXf<1|nI zZ^vrJYDC2I&N(QN6a6tXug>$W-Z`YRPZ|8%3`BNf;#W6Q#uISf*_f$!qZQloqO_tp zK}?m}=QGkLkwo}C#{~nW@J7a%hOeszBF3VC(#L)RsM*qZH@_*ffD-=!scXEp8P=hX z1Ql`CB-!6h6i!0pA4h8!TR#a6x-R`gno)l1RVZ!0Ex4n`Z4uY*^2zqw{vfW73eDoi zCS%Y-bX0_sSpc=>U`i2u%BBI+Aa?N`MiUdxfUg2PFOOgvg9rC9t^}YK z#l3+sw6n>ew^zL8XCfA!enFWX+y|ZvSYN$%vBryJ8knhw!^Z%j&y>ZY+CN?I`zmN5 zH5NVVkGBeTQm{16mp4E9wV@t2d^L^!eBsw?tuazMP+85JwY;iaA{g42edlkjj|+ny zNY}tBKXv$*;T%HtGm~g&IC1{o3QjeVu8}(GGWnrXse072-RWZPW&IMAc@i6~)H)a@ zZ4q?YKTPBk$7s4pCre13Otz>xOYzW7fBPdzgqFG(=5CY-1>sW13ZG=CH2Y|@cwcQ^ zS|ofH!+OjZ>zZtdTStF+TaJyUb;-Y4YK8Z)ZF{U_x(GG`yU@+t6gwWTN6$eK1Vip5 zn<&tf@V{PvCY?JrlgV$99aZlOQU7ve@R-yAk`qN&W%7K?vE*hQ1lreAQT`VRfW5Vn zqYJuFn|x=lRSXnWW-B$bu7{LH9{7t9r);f8?FjE^BLTh>`CdMe$Au8jY@=T=ybiNC zYO}~nojk)eXrM&DRgLZB5G!N1tCm_PYn1?ig8le#l#}B)_yotpW}``FHJL?}o|1W< zymRV}1zI11xi1=JrXhkQUYhrf@aBLj-nYFj4Ab>+ zx5jM&YoQ$O3U!+;jn0Idyd|Jtq|OsT%4tq&dYa+p{79ZMrI%-N+u|8w*?BkydaZar zNIzK?Z$5c?cZt3!D~(Jz)kna+5i&%_6Iw~**OZJ5Qk73X_}B@YtN-Vvz;Omg8vMAFsxWb;;~nB_C@ZpEt1b$C}!JC7MSt80K<` zyM`Zklm+0Z+eO(vN#>DbniDlj?&Z2z+f&{aTd2ZHT4zLzrltJ)m$6195^K~G z=?B|%y(vpzC&KP3gAQNJ5o#?7VGmn&)@nO=J_O4SR5sZyNY9(l2w}IRq2Go!D;FU9 zqiY{khQ}`VFH4aQl8&qLGLXHRxAgL;NUIe`aBUs*k6BTRPpN}7cT`5RL7x}eP@l+M z4d~t6q{jS)1ZA*pXu)mTmo&Orb;6S zPMZQ%Tpr?aGhbTV>CRy(sS(#Hvh0Q3#GU~st{u=z2 zVRXtKCF*NL4Vp~0_JewGq!{7v+3BG77>aj7v>R; z^R4*lRP(0vdTXE7sQ=^N@p*s$^bt}3%B;-zU-B>J` z&$`jepmq@${6HxeE@@UUv|NwiRizXX-xZtjxc}TVsSrDvCxR{&bo9o7g<~VBq&%wd z^-~hz1${ohG({qo@zmf)Q{n>gMR#Y?0LY=-qe%DL2!vKSyW2oR?d$4j6poaW%B5)A z-sh2*rK+dMWq-r9PnxVtBafwe{`twVaBb@y z>J1or@}b+=j#Wbae+*w_Pemy3NQ7q^=3-R(99O4v{I0F* z1>fo)z6Az(T;=`m148dBFFul!1%8g70)GA^CldrrZ5-}=3>+`yZIcAQ;f2y>8X~Jz^iB{s(c>y zRBYv#*yryy#M!o;-sIQlH^(V~n@dka?D5=yc7}dENOrAF`n{hg5u8Odd^cNPa{7>d zciAj?CL1K?IGe1$$9y%NuKDc0@KZvuYtw3w-Yfo6L>;YFqAWT$>0(dBOZ`lv$!vvZ zc{ni2PQ$DJk#|3Z3}a8{`V+1};z`=xN6h}*7|x3QJ#)PcKc_MZ)v)q!Dku`;C_p~o z^=j$rDrRZvHXrc)cxdr|OCjfK0L=e>i2P8*NPNL+`L*{~Y@^O8x| z2HT#2l*+_7L(PxM!H|cRT%Pk5&O@j8Jgo6iTNy@P^N44nUyS*)1?{HWhdr0Hu5iuO zHOqqtch5VdJ^DP~^pt%w;LNgw+bh6gSUaPg+haLF{G~PwDp&=Gothg!F8}K!^F{1a zs?xW*a>`Vxv;?bsV^#0ZgRkxiU$S;Zo5ZQ@pj#@k1loZWkZfwx`SiZCQ zHh&-Yv2D$B(}6J1|0|KeP4rq zyxAnt?eto^4y|VY$;vg%S8!@du-t9U3Me%<2zcpzzX+%E0<#e{E~M%)6~@8q94wlu zF$ZC~WXmnEF@XT zT_1zXyi2N0;Be}`5rd6KsLOvDccG?3rWINBHCt^oZm($~1|0;-l#{JnUZe<0h*NOx zkI4tOS{yAM_#IbMrP(+c{fF^BdL?z+o))H#bS=x;CAcmV|0Rp;#h+Qz)w1>3Sp4)T z*AR2lTcZ5}mxN@_Cn(|YiVJLw5O^s4C~QG6g&aE>1oBwV4CQx&E5L%p8z{xs*X0X= z@(pQJNyW<<+V}psX0pG5ZN?2l;_#Vlu*QMDSzfh%FPK$!MVWF{Vd~f?ggYgx0Q+d$ zvQ_nclT!PQ>!W^~CLujb9gW5(42$uS6Chh{X=$kBmYtB5$cAP%iY@tWz0#g%d=`^G zvcd75tNX>OjoL(I2hCIGM|^cNGN6_?s3wU4yygbGXk0~j{;WNPqdXy}H~1ZPjh7$! zVf$6r6ZnUM<-zQUwSpHYAaZBAvk38Hq9|-~(tnm({R)+(t+l2l^dW({R=$y#Vu(qc z=k3=Y7ieKREXg`pWu|aTQ>~rgS7U*%j-VH2T(PCiaJnlLQpa@V!CmJ&(8y)j3?L|37 zzWglo(kNDE@`32E>+B>X-0v5Htts>=ieo^v)@IiM_`i46SIk5@bR5PS-2an3HpqjZ z+myjXuWl4woWWrz&otc2;RDpy+g)&xGtKfRU2Qz=^|qjk9L4=y?=9IDCq=)LRA3zX z+p(S&n>Pgg0gtpvLUs?^UFW73ZJCM!x!cv@-m8*@;`e-)70Ff+7fctG$&f3m6v@4f zrbz)wyvlxV_mXU@lP1er6iaVZnd%O?xD6-oz24ckq@-_8Ym=eaSEf}KPE)z58ozUK zd&qe*yHL4CWhPMzR$VhzO%qm+w6DFIR`UW_l;>Gvj1?arjl|?8i5k?rTDK(^1DA8PDdtf$w(*u` z+)m;|r0c2mvYb5k^z5W4=^8Z}m{k13f=+Dnwue&c(n~ ztVG6NUwr;LJ2FtnnxKsHPa*eD^kKhj>@r%@7H2rbdjaiZJn(JcI0w5G@z_af3Tu`I z4;AIINrLMIfhNTXx<}CDYCLF{Wh~7L&M&O}DBYL?Ol&}i|Ep|dZx?&p122PR-G$Ia zs#;s8%2eK{vaHZHQQ~?|vN?H4;iA(B&zh%@H=!n7o)p7hl z+AMF7bmuf=#hf^~Oxt%3>;Dw$($cLWy3FDGsp|3kANcQ|1I+zj@aI$fzbg&@tNrRH zs$3tH!v1MNY|H--|NUF}dS)P(!~MqtjR@54kx$P$nY--uM^@1=*S(DVgq=5HUWJ?= z=y~4HfVx~^>1Rj1{kaX63`Z1sagTS8bF8)8#lHu-eyayEjU^153e)cb?jj1q?u>l{vT-8)g<}1r@ zypPo{8VsQL(qSfry4zDQ=79$yC(Qg{A^_wE_X~dx?{VSgV0{yLQwm%%j<OZX4UW41pj7U!@>kU>g=dPLQPceqzL>;r$) zab)Y+Z_R9(hqKF-z092lao^aey3Ls;;PQ_SUrAi%oa6PmHy|^~#*aP-MEVWlCsGf6 zX8IT?UUADmaEyAFnnu2S<1&+mOyIP|Nf(Y|Ixqy=-+?z??3wYAN~7}{{2V) z{-b~Y(ZB!b-+%P)Kl=9{{rivp{YU@)qksR=zyIjpfAsG^`uBfJ|I~t*fv75<1=i+t zH~7x030Rz!8pTblM#qBV>EwvPD@$rHS~{?_X~QlQ*5?4*K}hDoU16dM7)_~wdj z*v%(&g}WN#KGES-Y=%^1p3e}-2!5<(^(wm<_IZg*+G$3)6TKJ^k}6^;DG4c%#Y2U2 z7A8+`LX_t@r$=v}4@7h9?^Sto0zV!+F2ihJ^=vD)8|rJa{pSp;KmhzrxmhrpLy>ZVl!Gza6J*fyKE@U!b{FTOlsWy=IQ%oP69tvw~`;ig$ zb}y3}So9v+W%28+R}?7asy6eu1S+^|rz;@$^}AEFqd1sQEt3loFJdL*0eHU5)nl9N6$*5 z;WSp!7IV7+O60;hP_sdOas>7s@_rZv+xH}bV*P0lXT9!Fe!!_d{e-+MAsUBr?WyOt zuT%dIBWiSv0sn-@m3$Um83#}jbQ@&kdyu1HLe_Ipd>chy(Uz+Z{q^Ma%iG|a}v$BQV z0BQo0!`5a{7e^bV^w@J#2Nu-LY(NQOM^WQBi?%J)Z@C7!g;;xP>VVgDnD~1l{5-r_ zN8GY`ud5KSdU^}8w!8Y!+23BX5xfhF4BOb_k-@v3=LytlZ#tPiph@?!E%U(=D{F&h zV_uz>er44VIXrt2x*OqpXJPQe(E%extP+Y=``9wbR;KQ^lNbq z7n8BtI2LBq#z$d~LeDS*FKpenfiK_O6=+#;7GYzsREs;=a+>7440o^`uC5)QH3TD# z6@?j+X{#(XSNyA+qHi9#(iX;wj;msOivv%dZ-o#gkevu z=7Z5o=wXYDQO$T5lc~&^&z}Nks;-ojnrKAGKX4itm2uTVDzMWys$e3#w}*KdE+Z+V zsZ=^PhT^`#uK@D^QYig5^~{uu#9ra8#|(~&?tasel;tG_+FNfNlP8&!ZZ9Bw4)q| zGGwz|tSyUe>()Z|f{J=-9Jj^~0IYisQ)8fqPT)i3X}?zLM9?i~tv<5@D|x?-erl;1 z_;R*VEy!k!%Gy#NjyT37SZ-hkzH@ibdp#(2Tm>9Ee>2Xw?(a&^)3?!rZ3mz3DG~BS zpVhrAp3(hyJS^p3eJFAgSvLfqud#?0q6tBZ`th+s@!5F-G_SBqq z>}&X2v=`>t(_nJ3929YFqReq^w+0qSzXZSZ@>&={AQ>Eb;@-K$7gy<4Y|$g}UiVQL z2@sTLXUWH6eCilL+`B;$M(}68FoEev3lXEfc(BS#s7hgw_Fqrd0vVtD0};BBM)R;;^oFu;%dE3fyP zDS2XJ1EZhde7-kW;)D5?!_&bP2+PHt{gFh|gX+Nh=vUKm8+V7_SQ>~*fWqwjCJJfo zm&u91>iN&bc`qxFhZ_FNCPU}iE>Dn|MZ1>4(5^ksaatjm=_~oG44+$FjHgx&tM#GvG zJT&}Itybp?jb~4M-)E#>=()JEpILSA69`E=iwS?9teyt{W$d6@QST% zN}%(Zd!Yg!M+heyT|6Zg*?awJuO4pylJmZ~;?7N)R<5dyWWJdS%`oL)^2phEck}PU zYdeBf4--6Wr#ozKYm|VM=e_kDbKQIAKV4Mk3!s>L1(1v-Jd)8hiUhtP|b%vvWDVTxZ!9 z1eIJb13}th?xgq0t)!rFxaxc4Zt3s-Z`0RV%a zTLnz}gA*V{S)=j*?6tgNiOttwwF3Ko&Md!c!u4*4T3!AgAe!@6vw1J(YeQI98Q zYRTHuRjOdbZNR&G70CZ)PB9#3!gqkewr$^y%GxZ``5t)A75Ol&3fRX*VMmqyRwmkJ znUr+*^yFxYHaB+$(X=vOe9#0B(Z+%sxEaB4a#M7t<0A00j+Iuut3#9@irwFg;knF# zY<(@D>b>HuX;$(mIU@jbMljJ#gAb5Gd58Sz9)`>B7x$^->=_>YJ>ga)`Ez!5ioFM9 zT6dCnsm573Fzyjh{=W~-hUs@Wd?l_U#t7qn%rB*z#?()k{RULtZoZ=6QEhI_0Xw$r z*&97m2>>Hm_oeR9uQ=4OzT#Xgb`Q}PBQ|_|iP6<~hHi3~`tZ#Tqp4IBytez2WaW}& zSRZcvA&3}owP)~$ptYK7`M1~=!t`v&F}JkFN_TRsZVOxbesC6?RKFUM!kyJcXEM85 zRGy!PSwn3W3QrpZnOoX1$ANH+3GwbqV%RFp4Yw~>i}akJS}6X1F-ayeG$pmUVkLzC z6eYZk<;raeJe9wZ!@bKT@5a2hY9=}vgl<}o(-T`Q$~)e}ygV5lF+oIqz_ zC}MG`^h;prcib!|`o_cI?w0A!r_ZaEEp&};yaQ*9k#C4v-B`6|TIE-8_B! zjJ;V^P?v(D7ypnh#p>^?BowjO^Nm!JT8-$77UYLKGt$G>WP;gCLz|vdM+j#2@ne6g zDtnqh;arPI|Pf71`anVN(?8w?{B|v{Kxd^4;r9Eduk$ zZ$rQM^nW{Mp~a*PJKn*umf#df9H{Pdu}}>tP^0_ZdFD@MaxT#v9d&i2Lg(#sT1I~8 z`)qc4z+m5a^~1{?Q_3;o2SV8UzmACZVvSAxKY;7CTOM7Be$_uH_JX6)!uA(5v@`jW z#@8}^;XSo*4cQSRSsD@mDnoW|Te0VedkC%hskAQq{NKG?8-8?@_b}$Zuh~z|VrjUx zR(CaVC4t+TdaOuQMFfnH;Y%7;8iw!Wz z9ojA@@X;R>)No(UA0zG@!plLiUAqHvZ6hvoXkPbm*@qO*neDgyBN6tLf$b``c6z!Z zSEt5C3Js_JBn9bv;W3Nh;byb3OSCmWhq3rKXA9xIgaS|pBMq`}5Av+dils z%3REYi2{@qTqZ$V9Xte)il6EO;mm%ySbnzo!P%LAjU6Xbuavd3Nur)UZO_NDR z^gS8S){oUFyWO>@Yq?F$J#{B9WztU@HqkePAbu?ou6oJzb@`$-)A8`^Qz1?%p|nRX%+lcEJgke+PI8QxP8y8Y~!a2cO!Kp4B?0_P!4~YV;e3ai=<&IV1fC zCsH98gV7P)XWa4J(hJqcR7A5(ikkmH{ywsU6fd z$0i5^W+~UkvNG1!Q{U9`{|rNi3pxI(t+PR|G}@^*{5p9r`XRm}i_x{X<0&kCk{C0! zXw#JO5|M|T!h@Wb8`Zd|+$m;Ry3~9oq9>fZ@K@GdYaDvPEn5O{LSNXtc;5Lg2+|u= z=99G8FbLn@qgeYTvwv{lUfc;tP9(aSg~@`$XJxE*#9^XxVKwk@2+e$WZAzcw#ZHk? zM7@u1M5(YciwYN{a8^>AII1`G`UQ45BFrY3Z?L4ZTDaJ=zIMrXAFeXf!9?GZh!9|J zj!UW1v`-*6C+r;r7-OU+8l`>ZYODJmZRYB{T80&54>y3428Vjxr(5!9iZY82+kmp$ zXFnIUUK@D;cwwxBngG>Ne?;#O-`Hcj>WPW7)=l+n21PvSX*v}vZFjOADNO+9x?THo z=&%y;cdlJOORCRL{AR~~Dxx%{5id6M2^`wN2+uJ4sSLu`iLq|zuk#g;Nn{+uRCLdk z_HlOpdI~$&McnJE9xEEqZ|X|gGAah#(D%6&-6*g(k1Du0$8~}Vd6Q(-&GK!&zF!vN zkV3HzDA2$WH8_QRSgrRBD-8C13PwDGE=%e`Zx~vUvbfjK8qXGPIH)$?gVzj9LM!rd z{S7;UXTdCpQc=A}Q{l(>mzYw?9R2&n`f&J9-;X&E2&#?*9Zr|Dy^!w|hQlS&ursul zFujxIYC=BmzzxOv zv%rq~WUoMjoSqT@qTy+sJbZS2=VAGz%m1AxVjL%LttGF)h>SoQf`hmHmDLLKNStZt z1@DY)E6XSxX?yEcbB4$8dc37WsFYHY?Tu)_!bL2P*GY;P3ysNW!pY7?Tmafx-Z%in zBBo&_B5Be{fi8-L80S9Rt{?(Yr+FkPk@rpKZvZ3I6t+Q^C}4Tpx-!#E`2%OvV~$Cf z7JVsfIcROxE@J|Ovpbqi+C^;p?K$BLkv>uQnBNktE<;gYggr70K4N2_>0R!g2#V8A z&kEI-oSgn}@-DaA1tE#C2+83He92=*9nj!p7N2 z$Po}alQ?HY;qPYTGV}u}nulC`^R8-bJu?J>sYI`CnJW=I_XFl}Fa0*KAmMWF+lG)|cn1h7f@oQ`meQgnDLAqER)5Ti zUQrGG=!qVGC>WQtrgi$n{`F>lHkt*rO^ea-A*@gUX>+gH*!Sh1z~;*C4j3$UdO$oW zVlCD;+96&S-Ms|eL44}vtrGacn&Ml6|LAWP*b;<>YiS;v%Xj~d#?T3+d>q8^e4RYz z96VPMC>+F>!*HQBOakFXv3Rjx{}Fz2Z&%!HGAeNcc2b&+rY#X4Le9cWvp+dUHB7{$ z@s@8%QS^R;bDq|P8sc{bKSe)fP1@Vm?rGfZiPPrL+lV6^{`)B;`4th#j_SVLYpJ@f zGpPsRJs}M>fT{jIkw>@TXjbx^5q7-EL?Vz218=6NTdC3A_%XEN!EosFr%L=RD^L-q zm~%#VJu6eNZZCKR*eCv4Nx*Fd$sbH9<@5Q;_(`I`+zVpZ*DM4XyqW;uUv6Y`-uo}vk zyxDfnQhul5so7Kou@!T=64rukkS!n@McYk5gNtqsWg3D90xvbksc?RlbN7O6e5p9@^C~U;W3U@ z8Np6!c!)9v>%#)HebTYsefRZ~P+JL`IeH zQrny@=WhQms-7MTE|l0{rOi_BM(2n~6gr~j#YY*FJ?0GTeMDZQ@u;ehO)GmaM z*nR}N{k@VQS)KxW;E<=O5JL|v)LhrF#3|!>oIgLt5K33cbOsP*4vnKyzG${7g3Z$f zleOqpb$<-I-}>BF(5>-hbaC6_XMNg4*UfGfe3n@(-)|#&L#taPyl)yEy-aSw)we(n zjmpsKoMGNwa#LLz zZ8GQvV%9l(_c55K$)0q6jy}V>a};4$3w3bJ`$BpZq|od9HY5(}y;NqcVb{{zOM{+&K8ds zbh8JkEqKvuQ&w+HmFLG-n-}dZ?V5v?|0TvvpnL8%M|zA3@Do2^o}9~b(^sM?#unr? z(ylVLtL*Gp3@+H5opB9ShoUxnmOj>L{Yqe!GGZ2~>50PTbxf=)FFxNNJ`woCMDI*Z zXrOX929MB&d`x_ZA~_<1SSUs4xX(QpRz6hoN&7KEuZW)SrILL}?-!oi`2}_IY68=I z6Qa{!#oe5$rwq+(7)4cB4*mOM!wOqn{vy%qL;;B4=wGa1?HvA;n%UI}iU;pJ`a89gWKczGfGa=IQe3O%|pk*$5j*LGf_2nT`)x zR{mj;)({QL%=<0~Gc;?i_!-!z3`EBfi`>U_@e6J9RF0M~6Sd2j!p+?wDDLEKjV(0O zeG4cqL5=LzSw5sM8E+N;fn~C^I@r07#MMHDZDy0!X~1F*5Y=mGJw5<+^7>k>y z2hSj}nHa`Z!gBqeXslMYQHNRg@oSiSL1!KrHi@S2c}`kc5G=4pOd|cb?wuKwK+sz1 zzQ;(HiTuU=(r56kjsZ5e2uIelB97{xQC2>ZKbyk6-}-lXtH^ddA+8m_br{0cI!=!s zi>lTJJwO305zMYi^d2Y0Yp3aPBgVvnj}1El7qt?-PZjOYTLR6y*plooxJBvF@8WHS z4KKV!LXlJxjjq2)$Sh&uS_ol2Z{qDwL5c-CG`VPAeO^}Tv9aOHpt4sWz0!gHOn#OJ8C>fHTBs&_yDg+J-}U(Zu7f10~ANk5YtndEd9(sWvevt)Kv5rd zKmU-{RL#_T!V0r`K1If*B!0EKmKED%rLld)XnnRMvBSB_)rN zs%y=*5l4~>8f~0Ob{+B$6Y2nRjGn%la1kuc&>YP*@Wlp;} z616)>OBmxe&?iyk?s(d#aJvxVo6hGBD;&5jc$~^Nrfu&f8*eK*YRJY9NV&4Dq=wFo{+GM2d+;m@+tB3CTrtL1?EM`ZeUmc*(@le}mGPFeWXZ0<@js z*-4#Bz)w&31n{~}J*kRAX?Q>HyXm3g4c|&zd=%X(Tp}0JJZ~kUo&fLWECZ3okuuW8 zx>uIyl7)+6o6T$o{;Pw;5>tFK+=qF(9DA=Ck0OU1#vpYXDD@3T0}Y7x;P6kS7FN|p zG=}ML78=W7)op1zN(X(~hOwR?cox_LN-sla6#-K z=2)td5M~uH5^Wzz{c+xHcdIHTi>KyFGESSHbKkf`o^CWnrz#$9={QTz5vcwn#odJEqs-Fo*eBB|I^M($;x^y!(Clq#Z~G2M)XGl(HM zu2qTOevH(vkFW$ZxKljW0V+9u81$!oq$yJ4Tx#wiT+>$%TVP5@AF0`}1CeGI>}x_C zJYMce=iT2`A-&mN(iyp7RXUfTw2yv^G`~OP9UVW4A_?KM;Sb-|u9v(!;?{ zr*}$P(cD7vH9RMhBe;~;U19s4@~SV8*bb|7g)o;M!NAG~uGu!AKiL_46>hAgTj1RoQRkfj#q0YB0ot8a0a=?5$j5qCtPo8bl$J=2FaCDrKd)nQC z3uXJh^d=rJPuKY@l%w1pe@^jp`Knsqadq||p{eMU0ypP~IETn0@9*$Hza#<+D<2Ol z9PyQe--uBJXm#5|%1$=lA~wnf_I-S1Qc;=y=9XEMqp4-lj9bWOdC2+ApO7n8-cDP8 zRpyQbryG2f>no)PF`xJi37uNAlchA8xp)+BujkBtA#)d;xONOgK(rC@<*|DMR00uQ#&o=Q+4uwKXqMp2W{@-U?#$@^9h zZ|y|UwQLxu$9A+00@H(?#V$HI`j8+gc6d%Y=&d0~VZO z;GjnzmCHOeRt_P8-F1ESaPwlH=oQ8!@X(+BzHcNwRT?-Tm1fG}S=3#+noMu{G$`bF zJCyvmsHk5wxd&_`ar%Xu;Dk)LkAaq46bvdiO2ZV4UkD|8{!$z@-cl=$%8?@UG=v2f z=34k|4d=Yfy`z41F5bOPSoOVwW2Xc5nxpni^-87(6?ZMWTVeS9P4~D8Ld4mx$G!wO z#?N-0!-v5~+-G{0c1K2Z16Ssu#t?8I>+i#*{-@rWX?XK(XztL6<8N0UUf^69U?lnx5(mL5w;qH zmT0zSwP~4W*7aXbUffGr8fKbR0b@H)5OnitfZc{#n4T{EDM^b$M=1@{=Ax5L5Nhxh zTQZ!7ITf{u%{nqkYzu#>Unzg)>9TTrBa4tw5V+iqN~&H90$HOfSEBwpo0><-Ew9i{@HEiJMXTd2PD&Vz9mUkiy z$%3nqhT|E7>rm<~?K+%?m!3r!Szk&vI)B3qHhJ`H1}lg<8X@ zC*bBt@-$EGCI$XrTA_gVIx`o^7(ksD!t&-si#ZH*=lKXeN$Pq%Q=DcRg>O+U;qezh z4(Az@YS=%>)ksAp>?`1^*Uw{{J&l4l;H7*(X^tzYOO0&nTz>8~^PE5W+SSH{M_We9 zo#Q4-k+cei4t)@*yg|HH+R0VtUE7trOXQ`u+^N_y3VOppHd+8?)nF|tWYaI%^_ zLM(zzP{^y0K8a2lo>t$003j^!GHx zQOv|a%-sHFTkEOFidqqYliX0EtD2rQmP3$1W86Trt3-s2W5Y(R19c$F#f1&*L3P5+ z5jWDk2U2th$Fu3YLKe zG2uZ)T9HDj(Dp|{7+UY88b8i(-j%YqD5oqx!-6u5$_Op_JIaDCgsCBi_fNo+K zi|bhgHAM<#)B-GrD(1%VTGTi%Mq(IqKom0uPXf%cquEikfQeyZ6m3itt=%B9fQKJS zu^EaRUDiopS+oGkgOEgt9dAi$jsY!%%!P^jciI^$(EwiUaAqLMZ4lBcZ?tgF zAMu_y2r7>uzmp;#vOR_!Xmt2Fbi$Q6w3x^v7$hpN@7TZtn9m&KK^jD%m+%%rpwHOg zJy?0~!A3dkqzfl!UTuiIc&?}sR36=)$B^|ZZq)m1<}Y~EpP%JLN-O0R)+JLW1?JtbKeDjFue9g-Z_DL* za+RTA8TOzhL7)7rEL#=5wHu!ZhbcLr?WWL&FS79wt0H}crK(8$`-ZkJk{>BW#WnMd zs%+yeqsNIEpymF{e+e*ua3qvDRHVePs$E?RU`-lIyt1gAXiALjBR(8iIS-+Zi4KF= z-i@zGf27LyL_8__Ejl-qRqoN2{`M4_s0x0j*I4E)GLso>@LA-*UG@|a#Hw*j-BqwY zliR&y`zoi{&I%CvNBhc?iz5@F@b0+6$#nd{qhu!iNyKQa2}KbNMW5Afy4HAN z;vk^J(!iC80(XQ4^`+l0QKN2}#Z{xeN$HvMd2vToj6$HraOs_;^(Lzc#&C!23kmbh zG)Og8V#YeM0r~^6$RWwrfv7u6PNNONnQ|m>UQ8~Zc!IuXs%*+?v6ScTFXdH+U}Gs3 ziacmnyqI@-*Wq1 zIW=kA)^&d*RTc&N;wP4C!=TrvzeAzS^Fj_%e*89$=<)S7sQrt=D}Z%IW|zP(bZx(b z;7AQe=VQQu>u$nr?c$@j&sRf|!e|9+e{g<1%}?Qh$yM1gF~mEEh78)=Ry$AXOuv#*E>-;xCX9K*&K&vrGsfYcwu#j3x?LhO3KY5_eR(6gV^*ZcozGP zwSmY6_y-?Mo#L0ObfP16e0WGFb21{i!}a=HG?`mdWGyu8mAuhqK8NzodUKh-m9xh9 zN>Pbdu2Q~dXX7+tLAp42VLYrL9-8m_Bt+f~sw=i4H(8~0*7jtY7GK3Q7WL-r*EOu2 zfNFMz$UK?-kwof)((!ir)W%UOYLV(CBJ-uj;CwI^2^Jj&xxg5c`X-44#g&y>D3&~~ zHf)`&5%QDbGJDMBxVGw}Yy?W0QCS$hbWSu#gTr706pC|P<2!{}mjkUr%biJ+Rwd;$e_IuV9kbrzSM zRMg<`bWz1ws+n$P83tyX2n*SQ5Q%ZlVJd>JP{CMutvA_xg2c#FxzjI7X@f#EcI{qt03DKd7{Kc=KRQ(JC)oPr6jU2c2F|STPyGYJe zImdZXEIk!hJ0ac-I~|IFBTPE-H8i!N`NjaX!W@$ zZX@!GFyy(|XiarK5u8Tr#nq4{@&)X|b?RMr-ISrqC@L?ds7~oXQ5RwenqlB~sn|c! zQc@U)ka7YR;?`RAmi$y$a)o)p-)z>~o_Qb>{(Waxq-+ecnqzQl6?N&c(gt`1L3xKJ zn8?2=-!%W$Q4u1q*j3#yRI@Y{O(vbb;|e_lVhiITLmRIV3!klk0v)ijA|YJ>vUt;Q zhOuxgcP9WXinr&!)3ad_BlH`k;$6?T({*_8x6|f8e(YXBgHANcn5he zsaga+2(BtAhQggZiRKp1b1GozZszxkWnrGgk;IbnwgBCS`n1N_aYsfO;@}QCs3;up zDUxNRD&12W_132K*?~3!p_C~EX5%0Bb}2YySp>);r^{{vpKue|D9vr$vJVwcK#mgY z0vExxjWL4G5|Rn5zVZ_>RN*`1iLBy^Bk#EdQ)T$IZZ%bBkRWiP)wX$LIal-0aNB5* z%b0Sd3UbaSf`iy--oL^_p<5JmEFOt@+f`CWfX(`=b;zXmaoU7#(-Hj;u{5Cp7@)6> zE}8{EHMb3-wBg-<98053zupD{e(szP#0E$;Ca2ZTo4=QuKNQ3*nGijKp7W1#21SOn zDVflut;8j{9sJsH+11>=8z(CKH2L?&?Qr+0xiES{_^-qq2WCGG1B-?x6pR{&k!(oA z)u<^&S>4olhe4{x6)4?Sy0yrsy83EYKB{uC`UuWZFOWmJw_oEf=t_uVv?MJSmp~*Y z{x@)c{sQVlog2&fP);9zs2er3XXX>ckVU`OOIL)Or+NLaOs7Hc@K@(uxqLcNZ(2Q= zY!PltSi8hNA!EvpKbqcz<7#MfY22460l;M&bI{AK8f7CC>QX?LT9Yxci9Zd{KNhz9 z7E(_Z%gR~HKZPrH$u~d4zV9x7?xCanURMjhw&`(U+uM#n%eo^}mjeZ}%a2kqHVA5o zHC2<$wPGTdg}YfA6D@CAOpj_i)Kye0u3~kC>NFqVb8H+&s}X4{tA(t%Ce(R5gzb<{ zAtaza)`pO+(<+3ZuuqJHf1eI?>@>P1<5VxiH0X~KN_Bg}0l^8ke+F>V$=8_kj*nS^ zrdjTCOD+fU9Z|6&WVK!sMi9nnlYF_6hAL5-DTa(c1)*_Gh}8SnSh3svHT$$qsPpd# z+b+cS+0)9J|20hRO5{xCN*RB^AA*X;z+O|X1>N;rY&^(Y$)cSF(r{UOTBudFChaW+ zsaC`OaK*pE_XDf$97Ng7-|S~moMR|N*la$!eTUawiAO4gsKimFnV(*cK+(?QF5Rb@ zZR+8$-iRYqyj9U2msb=HJtGEqm(7U<~5SR!rLMk8xwZAavE?)Pn_S5mxbHv+{cE-l!Yu3RNjpG zsC03h$ycG(fr2MWUEbgpX{H@H6;SacAvKBKFz)zb+c``;GRMDVO9qn|HTvE~i6xi% ztdTkZ+=iZ!fS)LcB>ciUS_@tgM9#f3kahrE*s`CiZ&i*|N$N`0Vj8;eTDGz^!H1;o zjTbm(>vq=q5gO-N4VEcV2(#mDd7h#hwcN~E%NRLR0lE{}nCwc1`s(AIlN~ZFTT!tmMPSA*uZuO z7*g`giV#}F4JqUrfAhf^?;O-@z*kAVZcR+GXuWQ z3~Uq%YzCppY=$3mq;DI2i5VN<+%qj;d{-Qk1@`{{hCq40P%9(xDkCkNF|Ch8ryL*S z8ad8~z@j`13gvkG$q*LOXynPbXvKYCPDZS-h+D=ZOg7W=6j|xg{!$J zXFN?o8^Q(NQ3q(Dt0ZYS0>}i^4rRnBfTj)`D)ONT zwj-zwH&b#!@;ZaO8h<%i&@j+016y4iG#837RPBjxr}=t1UiAiZsHV4v@G`#=X&3nVK=fXVcBby|(50&&Gn# zYJhK#!okiL5YK@Y1+@)-Vx=(Ia1GJUjUKv^WDFebG$(I({u1wR0hpY@+sS8>pG3|` zlGUWbc>_4Wc1h*V=wKASupuGlE@ zHko@ap!UD+>B-a6X{+n1i_e-a4$?JUZ}l2lP4hLpa+n_&R&bH8ALu&B5RBZi;PW7)6X+< zB~okuCV}G~{5O)kKWg)f_ebE-u?md_>~6BEeYLI4wkTMsCJ~XaQ5Ky; zHbdQy1vsP1jS=xX<8a|FDa71rm!8NS{xT&O8ps$tabC2oz&sDmnc)e;JpTc~wM;a@ z3NxM+8O?~UM5@Ddls;>hpC^^XZrPk?XC{HnI7()0NulgKlvs23g&uOxGF4}di39YcU}Hy%~>tE38ZLWnz#2b?>V%eojIf6YpK5fk5OlQbC&xL zyyyB*jy$CCU#~pYEYsfA`Nxx{x2F&pV3@?PS;{gGCEPil;i(l+B(G{>LGd_>kOC?ChNMD0MC6&9^NIhZBHyn}V(`L@)pZ6nKcd&4h%KYS*FFT)} zqpowamcELNT)k=TH%6j62Inu#78+uw&+uSMXnBl{f!h&U@@6p2!B> zr`Ar1G~9cA3eUif{@H0`*C(HyKRbVVqWb*oRQ2Am%s-nvJtZD&#ui20GbUTM`O92G z;5~>k)v?xk5(mj9Pzlg@c30-Sh3)6;*ardZ23!ifBEMUSlKpOY(OWu$w(}v=cYp=^ z;+7`H9N7C?EAlHqGtm5 z-JCD#(%8G5aH`h*J!7PMAnHoVZ&0XVUT4>=LTO2Z{Ip)$k*AHhtPA@*ZAyMXP0$)S_sBbK;p;q$Q>NeOOb+$X#K>G#S+g_z>Y?Nni%45LnbS;SA>gL0>uEa=5pMB! zYbHd-1|&sU&$ix|dIQ2DY__Fcg2%&xPiyP8wy4{+qa$*HcOFYp3Q?VIJ$<^HuRauU zi9Z0PtC)Tf)8}Csrx^=RpTE#fKgm}-qh+@C$#>V-6OfWve$Pt*GCR4aCCZz&TBaob zdh+eXyC2?Oym@u1IEqII*S&4>(A}0hujx+E@Y3x}oN2e30pLxhABxkhY9)+jYr;N& zdF+1MeHswb%qi6StKu_zl)SJCWt0RICH!cD{epV5+|w0#Tmh$#t`T|6+kXM!rJZu4 zW+`>5v{8-N0NHi@wgA;ds0E6y2&L;gkQ}+O18gJL&NDpGGj^!0`b7$YTD=$_Ms5cZ*nzbvu#~gTa5KVGVJb5 zbr1lI$6M&d(4>#U#EZg&@Zn7!K90_>^BfSH@tXGUgfgJ z_>ije8b3|lGFcb8#)Wnmz^rLFw0vQ*w#h3rnFNwbj@XCl4@YO@=)+v9fAVTBkEY~u zRb)Kjm6Azvr&c=C4|r+vxhseqWKnqEfAjUrub+dMoC*YLX7^4dJOhx`%&t17y%ZC% zDoU~94*m*k5+CkWRJslr`F&ae*;u3Y5wlF4FIvKf*Zrcdo&|b2Cjx_%&vZOB`mVUN zH+f0gV*0>=oka*Cy85T{l--}pls-L!Cg4i(r*XCWNhLXYdNd{1MiM%6`)%*@Qk;v@ zHU!N4PmaimD4|6O%}K_n27J?uW{pV%r(UqLm>>)i6^KD1v5Z#miSTl0@(lB@jU^(UT&O1=H z#xlo~_~c?me-b%)^~JSr*6A-aXkN|}O0S&QG4)E=o?JEhQ(ZD8mqZCV_U^85qrN9E zX+=@S0S@G|LfZpTi7zvNUw}s(A!P@7A_2C)cq7sgLWT2?VcCCEEqsCBLp}WGuo*VkZ~iWOE1+| zAEqBeuH1FZ?|AM_FK_Q+b+=N55L4C(O1WNA#YoAj&MIC6>YBomR|^ax@NFeHngFT` zAuk!a-P7kL<2twkVY&7adjo!kRC8B?Zif^Wd6?n}8YFZ37`>-FqcfiIYOUmh)2WdT zF%1}oVP|eMTI8`3&xW;=oT%T?<(KeTY&fp>hHu_o z?sQfwP7;4stGj#8YWU%rmG?Yh0fj+(;tqFQZ@-w*U+ZF0qI*)ZYSf9|JYvp6i*}4Z zwK0jjoyJw5&3oiR^CF;n^x1#aRScYy}Ee$=GAnS z{&p!ICY6}fl1+H!I~gOOb|S%FVgPCik0X!tn47Al8bXXOr`dDzjYM-k*ujhoQ;5p1 zy_6I@1h#nTyv{Nl>#Ahz%(XO~e|#yTF`Trz8jvZ1>Kaho_c1^|&Y~Q1+W2IL&9QEt zX^-}G>-&)Q2n(|k5HSu&S94LSwOO{;5~~szg!5Xma*D71R0cP6&_QG0F>hJZzM*}` z=6xfpzqX!VKf^%@jMmi>Bk{rVs>Olcp+>;6Uw&+3aSMGBBmJZ6S~~P7R->(Z;$8{I z?BV=Dly^K|Odfc(oM<2DIh1xk|7}cVeBKJv>1QFB516HX^?bY~2ukMB--|i7r8dF}s`Oqy_STY%KxX(x>JG5eB1jPxLKpawESu6F;j8~T7_}zi5LYzMhU@vl#Xsi`D)Ey)b>!I^y=c!91l}UrPaDh*r zJb8jpUVr-2pNJ?4B`Hhzs(YiCFjDyA)2CDNzc1drhVaLtk_ZKgZ&f$62zvmV)b&5E zzkb_*qO_j`t7#Z0#kuyCV_mJn05wx1hZA!O;>O%K@Pum_gD$SQD0#7naSKAmySI zSoPEINX|{gJguxqEp(fdcgIy=011>35(2c=kLY4in-x_uF4gJ;WV9_?+{9(8l`nBE z)%gZxL}2n8(2!q4L#Y7`cH+JvZ9#*3>3d^l%uY%xC{MDg*3$%Gn5xZCQnm_Vdar1C zr*uTkk*APfA}y{MmQ}b%@6w8{-Mp<}TaTyHV`L$sB(kEV-t+4FebxqfLko#W{7Q(0 zS{DK!9-6EXsz>URsL7#QsMqFR%R{UJ$8t}@j#v6r{(8$(8LytRSr`1un0pwz2W|w} zy&lPU9pGTbYs~R{Irl<@I^)`Aa(ctN0+r!)kQ5j}am0h9DV9+ph|#U05>ogTawyr8*2)H6a5;yn38$QVoCLF$hrlE+yRBqt{?Y{H!h zAiB&eR?ca{PF)KFE0%1BDWBqv|^lGHWm z+Mwq@$~*yYiVswV+YM`Wi_kaC#$I9T;$HRVTBRJV8!|REP(U3#uk$@>taFjFt9q7kxx5~2^zVBZgRiX09)eXQ z*4xly`QU~Ou1z+Ipzd4A^fwg@&-k3bXi%M4k`UL9tbz1~-jZ)o# zjZO3bC(bsDuu$8LyU7?Ep#bCo{L+Ka0u2^PtrRij$7;@+c!hB3&WNQOkstx>+*Yhj z#Y6r;%k<)EFn_Eqk~R0cLaF(hsPE|vfR!1aSw|LobIuviaXc z9+G5)-n7juI_uj%zAi%umiPFunxnS zEkv4|Y$%W{3z{%5?pAZzgEhju96wM5>Zlv>H0JIBWKcamGFh?}g^ypVaT~zW>btlk zN|>t{f|;;#C5f1ywc_%Q?w@YCicl!tz9Ti^(+C z4YaY86^K=hDav)8u1tGJxh1iE;UzGF;TxQyFSitm8d1n19_bESki=-2$WV|Z<11cC zTS<+{N|F=&a+(x%4A0Pauh>eI>oZG6J>H_L?-(vF+gwm;rY=jCSK0c2GM62>P%?XS zwI`)lUE zuKi4O<2zGuZ7rM*wkmSwP*KA6r9n2Ez0r|*pZcM3%vI2;VrBlE{MX6*-~T*0{maS8 z_fIDO{`cRXyr05{-<|&D^k?(O?@v!pPQL%@&0lY>Uj3b){``GjukP@#pHIGLul^oM z$xMXvh|brf3L+5GhmR zZ3tnTTIOGLp+@Z0rnr z>H7id$k$RWa*^wMyj1m$u?>7MsQmzrq6^cv_4klWaBWAjF$F47th1$r6F%Hb#9Le?T6=r@cyo}U zouIhE?#$MNHl?04_cxB>8*8l5$6m5dG;#}%LW{>{;EI7nkcq9$-KhLz#^xBXGQ4XP zEzqxZI!wwkRzU^}v#|XbJ+l(`YH_~L$u|-mt;6n_`Ex=~TiKyfSBYjU#rznjI$&An z@{G(*O^DzyVW^o|uAEv8U_~mK8eqV&DaDG&oECB^DllEpf+oE3lrSu5a;F{zpB*&s zj3lQzyxE<$c8X86EJeK-mfxs-I?1RsquWbYYsr$N>zsyDcQLL`a-qFH&<;Y$m4#kL zf76SM@)bFejFDGnN#ifqesLMCk!3qpOxQwEIbwl1OgZFp*8u!#1bQ2JYL`{-&%?{1 zgg7iyTY%-MaN;^Ht}dmk&Fi{~_y%w1Gm$S`ysc)3s^cJs0-yIht!ah_0XBY+zLu)(~#iVP%&C~Iatq_xdVy#zX-wJ?k zUf7C>N|*$rxSCq&E84o*M<_4Z0EOGH@kqqV!%7br(%WO?_xoo3+^|)V(JDTcD$Rf?q&xN`&Yq+_-LArtS7hq}K&Yrn(x>Eh@%@;{NC(^`w1241+7AT+WHC6LlySC7T#=D75(d7^(&hDuj+= zS8rNUjO;4XVCS*ubFQ~!HfHrSI0?O7pY6z_b+OLzdgh@%uHP(3L zm5GVQh+2PJ)eNj?j&;IHhEj8c0*Q$Vz?m@-tPr{(NV25) zf;Fk!E1JR@`O0blA!~-24LVm6Hm_h>W=bHj9`TM!9zy)DORka zDXr+J$m}=NR?gB9FEC(EBure+rqu*Q!6B{088Db<7$d+^s;1aS+bO*HAv$PGY$C!qb;*ax*Ij3H%;Py(nQ(9SoF!7VdteH$nuJp%-|Ye5pfvk zI>&|G9`dkZTJh`!t7H?LiRYPOj9PlpmK!+iU>br{sWw*hD+V};Qs{F@oxY*Yk8#mq zJ-9a_jq~SqO1Cu+H^~>6+#|ufg*QaG9acc4>Q=KQOM1V(rMZz_C$`cIG-(2_n>>wi zi9|#cnroz|Ehb;pT#83sN4teO$UpX3&FXdBhK&r0C{JZB#uW>CPJY)CaI`RNHF`|Q zIWa2!R8NS(j%wc?37lLHguchk5p*Uxvk^Ge;UV=&8q)e&>U5YGYB^GE$?LTfA zv`|c-;tIOH&aC;zsw#PJt@O>y$Y)d{nxu(K0JE2gN;L0`a~wA9!cxdU<04<s#lImb*;^G1?Sp9w*spp$%H=lE1jR)J*y$7df&$ zyf;p8LrYq(a)V^7^kMSJV^3HImdS`mJWthm0LLO2GGv`3EOIMpv!PoFxj6##@T_Q+ zESV%yL@jqhcrGiNWlgPay#NGAoZ6S<3Zii=+{wPz7SJN2+C(+4^8`a3d9@xBq_?3c zfK@}Hm5$HRlh9Zf;#jI#NMM;;<0-l{m1jXw)@XuXinxfz8BOjUXqid?IG`2Bifv~o zwYdUTetaCaMVo2nc~{PKJAKmp7Pn2iGU}<7eI>Isz?wmbBU)EtMJt|w1(44j@7l^% zsW7n=aa$Bb5fGO2Wa&NGFRezd`|dzGs|%;b8H2c4vg3qMh*}=g4Zgt;67Zy#EOK+M zlu}2b4DQ@uLRgon)8LJXjFS?6N4^6ex zVIbu_lMoDKl74*U8Qg2Za#qp`9G@D&lg|m=XliRKY&{U}GC(J@nbMJ*+L?Au0nCYN z1(?knIb8q@bxOX`CE)G(V)L>G>;rK^5ZVyXK7gJZZ6cVGlZ&iaI&Y3VSS)X{sjKZ# zaoZh5@kJ@5d~3)KnL$h!Vhs$Yn~$3KqbAs5N;b4K3cT}yuBSPxG#-`l3A{7!{cA}BAmr|))iP(}8LnUuv$kk; zCGbdX;x*v;W2qwq)bEgSJc^0gk)4ujW@_fRoo)w#MblOsJ7pE+nQYF|oBs1Vn&^nL zaLlke=ixuSb;~BgjkioS`K3k%?brEv;Dsk(N2!bmTqW$1dkS9$<6_Lrim14E0F-E&dLje|~Os4J8(WTkcWC12Re zt|n9<}16UK_@cVp`R*jkS_MK*Jtg+;;8{A#q|dM(lAl9zw45bbMwk*)ltpY?mQR zKGo!m_O*v(RIp~)$#Wa3E)7%xMudj*4lIw0MN7_xF-vvdnr*nCwx_4DFoQt6P<%+90#LE zAc06t!agL7rMmIIK6W$ec7yAJ6J}B83(5|-9ue(Ir;Dj&~(M!$_qbspuuxLw$jdiQ3>*DRpcs*)F z7$e!R6zX~J>qIm~g2QKN0H?5Fqt;qa?$|osI7oEhwJsHG$YRDptGw9=l3_^S_BD2K z;uTfC%Dsk4S>fD3Etv~EcgIo=pi0?ev#_olenX?^+!A^TC3HXk!1BlL> z7ou8;TrPQG%J;%P*Qp8EBCaYSc^rRnaeor%o`HVETHQ&U1L;j5C7Fgs}i3B#c z@92)LkIuS$DVpNZWq#D)f3|p#mEj!Zij}vVVsW#SyAJ3?a*nQWjb?q!w4Nkaz{foKyJh(VU1!4-I5|Z4E`V~Cu27{fd#_d zmP}_$b%;8N%#qtt&)H@XX1D%UlWY`4QVGv=TEg}CnyU%ZD48q-hUoSV z!Uzdw2_>yf*j`17x-5j$b)UL>CXRD*i)d`pLosXFZV*Tz)oII76ffEsZQ8*Ez*E|{ zEcyo`in<9UMYYY=y1}MmV)^kg#@%R}dhAM7s#N91fK{U15lDv*SGG2nWLSjTsh4ig zl5t=%?ge;UGg0Sy-5Zpz-A7dQ-e3yY=IPzgBnHMca<0ZnBL*1F!0}(sSy|%T&DQ;# zE9y0=U^vt8rZb$o{us_N*>O5Tmy#=n+g4VzVrN(nah{p7Dba_}N~C7lb*AtQp?@IZ^T7hT$dAGtrG^?2n_|evNt}DGzj#hAJ$XtEa9PpxYY&v=-E6 z1b~dW_@KPuk#s2X6*5gs8W0uXd6LyB#t$J2RlF*O&u=lLTS+6$qecca z`7SoJD3-zZF*H&ypE8-SrfbrudMxYxgEhl}Ns%a!$6^4vL> znSQ`am$#&wsVoR~#YvfHst8 zSlMp^!Gxh;DLfj#1 zZ*L#AkhaAWGhsos!33x@8Mo=VJ$Wr?9{a!&4YpEb^@0yVV|~`9uj_dI@}-)OO+~TRiQ#CtOOd6L zun$#9y)RG1YNZ5OOK(nS?q^+5s>8m7f%I1_3}A`-K!$S1!(bk1k3#F~I1SSzv}2k5 zc3pnhnJkwaYKO&^RjmY6^0rW%z>M|Il5L<1G*I$6$^C(g?bfxjd5wLpw*&sIzd{Um znnOB15;%iIWc)09s*+=G{EgK;^_B^U-19Lxb?ue$);WAkKGkR|o49A?J!cQ+I^%2d zz^mm%Co7+W0D1mf_$O#>K&9#;O?A#D;-zYIhN)BH^$b#b&PdJEzl5w}qePyd@nRcy z;i1dF&jGMEu56ILw`0-JUuq%Cu~ZCOGX^Xede7?9)Iiyp7iih&k%*$I;=AyI%W6BNr`L7AD zg?FnlE?E*lKaN`cx<%i7bNO=YFsWyZAPAq^Llxoi12+%9-Sx428pFMS~c>V@;by!Wvgf%H`cIrMJ)NU;Fq>83pr2O{kcr((=%v7Otx;jpHz~gr$1a!8K#rDYF&EG{jN_08K`<)}zH7{MgY@AgA zwdAiaUQNk2T9whqGIBN5f3>FJj*mI8QCoKh3qg!N?>_@Ny8fom;g=XNzda$Wnc#}r zTBhRFJ5~&a8+Ifqyt!ll<(2wENDie;X+gwIA7Z*3LUOp#( z)o@$}dTVwnk+^b*x)Q_~o?ep7qlO$qthB)9q{Hog(BX~Qg=5|%x0 z2=NxMTh%kSu{vyX!S6Me}*b>2EKc}dpmG9j%bB0@ul04h}8<`CEXvo0BQ~Jt5KDsb%`x$@SyaO zc5WMW{_fSq%Qvs4t8^C@#qzWeJoiI1bn0lEN6gwWUpSS#H2#ubX0qd0i9%#z(O-jk z#6bKOgOMk6qDFXkwWeB*(P*lr8IY!DR+_Jt(b)vQc*plF6m~}=N9hm zF+lG&XYH`>Hg8Nl#xiH@c=&X-?u7C=ve+P5Frjm)F>AAI-4p`U))BhE$}pw&loTbu z=NVi041r6YFB}vt7n@7t)wK=zZ`5B6&cz^M*dK4dzJbEM;4j`;(NPWYED1s;lL?rV zPX3`}<}^Dc-!c7uNN=c0o>b&vDQuZk=v#P3VXI=jH~sXvqM+nYJRVzgxH|u^upWAk zcapES;Lyg}ZlywJMNVT%B^f?XiCnKTp5HY^jOIdvBr>9YFzdHu)Sq9_Jsoyh0XuX5 zYLt~349aSs{mrlm9qi#M^G8R}yMV<03->%iwcml~a+ydT$8h1+7PrmG<@Jl}%eG%+ z%lPSt0BWy-JmeP{I!vKA`;k?XB`q%MYWb2&0AY+5P&S@)axkz`u=Yj}t_Iq>P#YRA z{&sOiXst%C(q$TVlnk0aUSp2)=pO3lcIcm>aERXRH+;V3#|l{9X|;tP%TH6cY_uQC z@4xy8q4+zVG_a%pyZEkn&MQtUQI7H(w4Smilv-9|g*yz)m^_GE@fM7Em-9`6O@N470r#C zk{{#Q9{(}#SW2<0|FfnU4&U2-nEBenAK$vxW2T|aG`zL;l3E6$y=bm`R+TiDYTnxZ zl4I@UJSHbqQh5D)d(VN%>f9Aqu$?zJ9j|Q;jIT+1b^^kxj<;y(`|bR=?Z1NG$ZIYk zG$&M#9&YARmxTq%?Mf+~y!e5pY55$_;JKEh%3#8{DaAl@>%~5))`hNhbo1g0s7Uf} zpFaUf?z3l~Kk-if$IUw#ZAtp5-o>58nenm3tr%j!rchGWHHOzJUaB`aEWQ9kNSjk5 zUQT*tyn!qoNNkT8*UjOj=;)_?WVcamf#bc4ImRc&`ChjqQ*ADHk z;I!qjPP@G0uFJ{WFRxpkx($!qumz>-soPa`%IAY4a3Fi9gFoF;Qq`b9akz4NNZXvJ z!{iroN2#9UXoNgR;~i09!D8p&x~LHi3ohlD49sJ^-^eU#{~+@z75i6=pGg^mz1CfI zzv#-^$>2ND{n(bx7kyAfaU8DqQ4JwvP?Q?}=h;>|gUtXM^m|*w(BFj9<2{B&aj2&qR?Mr&8Oc zw8I|#XG*Kzr%8X;ty-ib64}(vX3Q^NY6;V1qDj5TNW>~5n=9?$CwhAMa_fyL%IZOs zV?a9Ek8~nv^l#lH(qV1n)7pF2u=hIk_J;PLUm~_j>&PwW_k&o!TW1S?T3c`XQ!;?c zV2@32Z?-~Z*jfL;B|n1amKL9E;j)|DN(RhzV zs#>83?WepRW32 z#w(5etP!FeUEA*}$s4DnVk3%W=(jj_v7CscqM0de$*Kp&{L4e|8wmw=dgB6wlE_pt8pl!J{oWTH&@Ws*;AXjTF;{hajFsGnh*kImZm?gZ& zo+QwPGm`N;M&=SqkNa~;42E*aq!RfxdNsbNSzVHOL5TM2jEtyh+x8IY&1{Mp>FQc0 zkQ{kIU^7HG`C9M1vQ%*zG=NOQcxWc_1Y2N-#S6rmSW4 zMV+Tv1aNGtw?{8FC}3=_PR%gotfTOU)g0YU0U5Vy=-oCYZ8u*QK~-uD z07Gwrt*7LDP5@W=3~-NQfXuZ7wb>qL_FXLH7_8g(c@I6?dbRzuqiANTYQ;7+rz>`* zxUB&=ZleWCzQ|dL_L#V$QM6OD4TkV5l%drbqjtR{bg_WyBKI+~lq*J&@t+zD!zCkE zuiiimIEchb6suT{$&GQC;A#OjV(^rZS5j1f=%*;_36d5V9D+$EvrHs@I03v8vC=g~ z@|3_sWXWi1s!uDis3=%ECFF(LjB}pQDyo>FdOBrkt>kK*Bf4Znlqo}a{5eS+g8B@K z++J`JaMi*YuS#03QFt+PrL8sM@(@ z-Oe4>`gXNDx2fJawtnYU6+E}9;knHiY*EK^*GisS&ho*kc^<5u=YcDF9-^jad{xge zBz67IJ=FEwVqc7YV7Ka?adG2#_3FH45pHEQBWOC(DAii498gUiH8zs6S`{tTK5p7Z zZg*ke#>bDLIY1+oR4vbdbNH z#YmTuea@}pKp4m4OP<7XERKbF zHAS-$>|~$Cf;u#pWK~^Ya{*#N^_EGA1)0f;mUk?*l2z<1U8%vd=84iAdJu(2*DZj} zcth^-bb5^EW)CQ56pc}~h1M2+Lko!(5hV+OB~}19Y_*Jc$l*mJ>S(#wHumR{w&x+c zv+k$fO=B!MTPfa5@<(HZYVX^r!XH4$BbsI5_C zeGy)rD6Ua-JrQ3q#lX!K8v?e`t*DT&?iH_x;nj`~b#O<^mS#4}BkW8GXrqP0Jg4N1 zKJZ2^9;#$uDYCk<5U5+9oOnUUi~$jmtPRQLG#%Y-eI&*o!urwi2gF#5`xt9`q&kGx zmX^d=>-v^vz#xc3WbZyQ)C68*0Vg8K#=uhG_3=!N=227j#KO$AB>Bh=ijpWE4a$!Z zw-LTB?(iRH_8PVDcd>?>;QEWJ%aPUu=g9$kf^)~O)}nBU76m7OiM(vO_3aq(Ct9Q5 zUU}MR+L!96Y~nbgj}3Q1E1^>sn!~p#4jS(i()q){Xbz~50sR2jQTJ+d_jl&>(^gfq z#>BT8FJa(i(&yleBQ(!Ng~2dqXo?@@NF+9l*v-o>o$Rwwvg5X~R?YdaQ_za0VMa+N zOSYn;0%>Ewd?jsG$cF`YpC`Sv*Bf~j%_eJBin?6v>b!2IguM#jqvDwXTdJ?Xko}=gV!2^_eC%U9puY*JqZX`f=pJ*0e@b zgJ>g?)Md%?DqDYK$=r@#d8(Pn+fhMUL_oW{sH>u`^jS4}DmAM!^*8M9 zJhS@JXp&05dK%7aY+nVfDpuyt$$y=^|NYOC)4!aYeE($f?|=XO$@?jM_}%GWPJcFk z{QmUxeb)*>CfNi_394)`uXI0_UiAEL{3lt@~opRZPm`CudpyC1TYO!ab7eW1!lvI8Y5|x@8)G__Hx>jfhgi__Ii&u~BN^ z;*MVWk5Ovj>oH30F-q+*N^K0y$0)VOD7D8ZwZ|y6nCmf0?J-Ji4^e8UwDrWQMRrct z=V}7g{yW8~NwvTdc9A5a&Z`@7$07nyI<|xj3Q%IIYg>r1C4lu|liXoL*l3+{rN(cu z#>}H`amPGmsr?~xiD_<9soqY>1u0p=3$9@jD1MKtP6Hpu@>^a>md#NrqImDMVnaCy0oe4_cY@vuhupq47ZokJ%Y&W&1Dv*Z04NMq9_GT zmP|S=KN~UI8p7U!+h+BG=cp6_WhEtMYf_4eR?#bHvy1MkGOtuGiKK`kTUD;1Jd4oH zamIl=8zJ7bB(l_$_WZXeO8-stZdYdv6dS7q$>QS%)af*LTh)5Svpfi($!Y<(KUXl*>zBC1qz`X69DWDUaqJtH<=&Ma(W+bNQ zzu2OJpO@mE$_AHg!DUsB84-M-y*M>gk|@cl&MIDHc67X*cxQ;|Q_}U0P{;I6v9b)7 zbS)_NGv5>229{S)X8zuB1;oG7V_gpPbmUd_d zB4yDujkwGPIvvHTT}P^xBML^Un?w9ZqUpcn_X&%UMHS6mJ8|zV-L%mSGo8+^&;Y*L z?~BNwnPMw`eauuQ=`0itbbs)I#%Ajb-r!}Y!E?5p#Z|I7E@x?*hrOezkmL)1DtpO5 zXm`)r!Ex!9@ZBuc+go6ieW0r%V{&@;Z(wWFd+NNLrRVQnUA%nrYPw2yDKa^-h}N;c zf?cb)v;VA9L$MY$d7y~1s4L&y@ZvdnmCG7R?9i(7nxLEptR$`u9(kn)*Dq0iglc%Z zmXDVT^X)3sv6oV;Mg z(vloq^?J|!j|e%5RFGqwNQDqymqe#zvk+N(TfsNDtDJ>X&H{*fpU5)Wb4m7Y!9hQ> z&-)AfI`!8-+AxLdtS2R_0x2NfQtSE=(9a!5Bdx0?%PXFsWmkp+4qetoF6XFKq%4Kn zHziM*B$huli_l9l=h?>?-B|Zcj01O^>-J+iWyiW>qa3ZH_*~Esx1Dowl)iWj3S)~w z!Z#ZXlBO#j$dCRcSY%iw1G#GOS97RttC7isX6*9oD6LH2+g8l3^_Nb4=71V9R3rQ= zyLd=0(qTuzV}0E}Q%S6s!D}+1FyhO(8I~ zd~jeX_u|n$%0a(+IY`RvRZeG_-_zP&(Do7XWkwf-!5`--m+D_~ef=6dDRr)O0q-b) zZ$a~4n^B5PVfVd?jBZrnZ|51p^VAdsb@1c#9l>CAYSV8p*l5Pc&5|c~SB~>UlrsN=sFWh%!9u{W7Qo8Y9AhaQW4reb zgq`?~p&Zq73Y*#2cThj;a86yGo3YusGq|oo+5SDQqPI>0p6Qfbrepj>Cf7_)wh=p* z6ty5b-Qh@bIH}A=fz;iDdz-Z2IQY9gu!ryPc_+fkk2fK<&)m{+tmAjuy#&0a5JjE& zg3z2{-r4_v@887@<>c7J>)>u;PJeb+6!KM1HkVn|pg}2ZNxy39By4$U#sFOnC1`mG zcYv+-0ag_SPNkEoVgpZ}eEx?&Ovy`LLV+P}D`+<^3tBP|Mls6?bkZ$?e(B9XH@W43@r=qlADN3im~9@;BlpvFbIkXe zJw765+|35}(u~Oz-^>9IPS;R=cp&rJ&;Qz4;rppdvTZGr2uIt1c2!5GQYIR9Q8{#y8uEer*J8?IRs_qCWag_u)I;XEf)M5aXVRXke3@&_qxbW3#+LM+ zi@HS0l^{p@rz3S%u7w}o+gPTXn*T7GvS-fhYKSm0r@|QKCQ#>pgPff%1DAhbiSb?* z>a6|Nn_|}90`3!S$?^Uge%e-U)F=m-!D*67ehJypFE@rPwExs{qxJn~JXv4993L}) z&zaLl+IRu=J^6yl02_QmbGkqy!^!p47jI70N&)N9<;$)I+LgBO1#fpxYwQfcsMB}j z$pZ|-9x^x0`4uWn)pK%Qbpb0dIKW`zNW(&6PS5*Qdbe9g!a)H|tG8pYzBf!@E_#0O z(j2x4J+Rs4Yue=i!9atf|G;LL*`B8)<2fVqj4mLgDMy2xVoVnqE2|Z(rJg)8KM!88 z>V?R$`b9FJM|M0eANZ-X{)KB4#)>d|!rg#=XLvb#o4|OL_>(pM|YnId{ zuhz&FNBH(YAjF!liq6CknrQxtT?xCWI=v#_?z(QanglhJs{*rsS#=D^@L8Ev>Hz)qGkgsfmW8Lojb2MN~M9)xnGbh(J=- zGyF?}CIZ8B9r8;U8%N6SS*F(fbR~Y`Sw^R#T%5Cf^363`JicSIb4Bs|Z@9nC2j_7g z%k6{S5=s(A*Xc#S+@P$?=ora7VSPv5vEwx z+-H$u5#~a`-Z+0|EC5sTR#Y0(ME}Ai*g<_FfVCdL`FQjCdP=@iV*1VNYm^vGyjK%f zex}KL$QnbFPS8k^D7)7_VC1c>y1~hk<_mw+3MHk;GNn4yMM|s2`dF2Gv0$ao!Vvb+ z1g5CsD<473^HAB}CY}%oo<#WiUu_=imwA*MEVa^wCs26^4vw8T8Q_tRFI2%7+7*(^P z<%0R$k%x^6ylKJ#u{PUYqXFaK=Ra+i-$4N7a!j?pL+^64;%c7&8=8G!yZC9%btAzTJ9yA7o~=1rW( z>yhpJ_hvgAB`6~L^H0xnBFHDaShDiE=9NF6&fN^gPTQyxNQFos1Dr|8WcRoLeXAK^ zn$^uur_UhloUMv#eMbH&tpB}DTq*UTQ1Xj=$}>=!HlIyJQmd; zBBS{Nbj#vyah{axq7vs-CeMFc+$|=F$Yn+I%2R|om_;%|)%3;nCK+dW^+Q6pk*r@_ zYzWp!&;8n`#I-TIEGWgJT<%_va{1!{86x^9mp{tok8=4V+Znzd$*&xw)CLCm_S ztEE5ku)Q6wL9!wR1|K-$T3ZmCLQb+z_7voWZ2-oKaVDqaOUM!hw^~Jc2G|zOeIo1; zjF6vSw0b_g?jzQFQkVDaszF7I9mZE#ramV~KnJ3@(a!`N_T<%zmw$WZ>n+VwxukdO zOg&K8h7eJvtem=f3cjeVpm-<`sV-3itlo9`@|Ral?%4V|aP|hyU1R-M$J~Cc{=wt+ zKu7Hs$803y4HP~Pf1frN~125!1>26+MKm9D^O!h9570*X@faiPez_Xn?fJYNb4==`B z1Yd}}Dn-_TF+^DeIiRj&1xNd_uRALtE8U#3V^};REe?MyJ2(4mMGhInFD+}17N-p2kdJOQiRz)$wD?fpF>P`u~I zfgfZ=tGcsx+GoofHE|`wik~wW{!~9^jvO~0f+SfjtIb3GohfJ6doC?_wXA2;M68@6 z>Rcm-R_pUwCT8a=stBtCt19`du0$y@hUi@K#e|m0l2XMz)f=?2W-zzy<+AQW*rR_CY6ZGs38x9~AY6|8qr;JVvv;Gp>0Y**D=p2^$u7)`k8uHyas-;}0(|>Dc9qUI8^e!0h zK$n$4+rM4D#}WsrvST^a<6ut=^ZP5yH86Hmr@r2Jth_I8RZXxeay@BcpC z92+m`e?RMVflDH4a8&B}K`4N#Sa#J}z=LF4yufrlw7h+_ox?y*C8IKO6}*F6T?GEU zug|`te4sVgt>fD^X6sj19=lijv~WrVHnTrtYHybeJ=9ya!k=`v$xnN4!!hjrNxi=B zWZo`qucLisTvnih-LZAg6b`(7lLWqSMZ1-d3#!D~z)}9LVM|6ip!u56q-tRJ7^(A= zl^GP6$K$f^kkwWLFp}Zy7OYg8j-^wg0b?>&F-zP+Y}(rf2D4w}F1HiTa*A@hObw#b7-t~w!U>wR)S=Md?N%oTXLzrj z0y!q#vO{~EN;qPd^xi;ANX1sH?ynA|{dK#aYfU!Qj?VA6p*vre{rC2)mhK(xp${vL zo6VlU!2D+LDU=VMgEo&}V0OFvO+B+RydkU)HS&M|-*5iq@%nFnB}-aX@;oV1_}`=u zsjb<$rmJip?|Aa$$&)|*@sIfL=lJiFXMezdpFaKk*`NOSFHisQ`JbLWefs3-lh6O< z$`a;DX^Vhe!!$G<0}XQ<&99GfeVzCo(+r4n4f>iqDUYky0T)g{eZ?Z&fW zDM|qTY~H!jTC?Q&qRwdPKJ*)cFkR0Z+$ME+ny6PrW1>gs{wY#PmTYAMF-5`hi>u3T zKf7*y-H}O}_G?#=bcPRYexRs}ImjoC6wz-)a#owiKC7CgMU&J?_|jctsqP7CU0gQC zhdw)lm;va7_;){Y5z8tYI1e6%;c9olH?(QW(oTf4_;ipCR~*}<`8AbRv1i^&2ijqZ>7 zMn^u9_c^(&8pwGEK;zHW5~DDbHgM*?9wN}J zHg>yU)sShisTp8w6}N9i%H*vvNAz2k{H{EYedixy_FAd?p*g$(&NyS42|KL>DOpu= zraScyF+<2hj6JzEg!?+``e>1;g(VmoG(0v~1Nwk%MI(|DB;?{z)47Ia&4MS5*LcF5$oXlu>dxp+o7xE=d zn=bt4`=c2x-yii2rkxelH=ZvTAilVryqtIQ;sGg;WodsKbxKTk{ItnaQ*NOp8`N?T zO8Rzgp0-fU9U;S*p4fCTCAa@!sr>mIKL7dlv}eb9%&2#a6)#Is6fC_#z7Sg4SFD`K zq@)ES8DH_rY`n%UqZtfIk#2vjr;@=i;X&!d@7=Ki@Y`2xCCc@bJRvJab4hZ6uhVR6 z*c#l!$o|mJ$ow7wpWOo6%}UN}CQ_13)!QN>2~9I0^eN%)Xx;q}NKAe$Dbl zwcNV@89eUvAF*G}AoO(H8J16C5WZEsVc(;G4Chxe43_a+Yh!m{2+VCx)Fp~84&rXl z=^*=9xa(n+JLw+pIC*uBB1BrKUzmEZytgn}}x zbOeC=(u%_XMrZ6BpY{$Qy$k=Q|wmk$Hr#LZol-G<~I2 zpwMAux@c4;NeWt4_E2xm)osI6t~Q#~Nb7l{sYzDetC?-~W$(BcSphM9kzu4hl|zq9 zARdL*)l!uFr}zMdqubUqw+-MB?$QhQ*AT3rQ`o>r!w5dA`ezhnlo^1LSw(e_qLM3h zNCaT?L3|9AAsX>ksBS#F?cscJBiCtuGzRXqme{EYJEtbVSjQv!q`&4u zgiSZ&*BwhZ0osWp9%`|vY!QiU2TNnyDv2#RvFfpa#ctylo0fK%So`Q``}l;km~^yI zGTK4wdiY{&$Og}LvA?>R{p*5FVXKBKCvr&=D%phRl0oduJ<6BwkG@dc_5G0np`#AM z1lCmb(tV$sc=_jw7NaFA-+Lhx_fhu1o(PfWUUPzU&phZooP5o|ehOz^`=s9AxmQpr zA4KWh9c^}bYaY3E%B5@`flY!~;P7L%4Si>wkFPx*c8`bMZsYfO*bTnxK!;r|Ss7Zj z9e!gT*&Dwx1Rwd0z8?9_BfshE(XXE0gat-NDoR>Kb3y|$BdZbD! zMHZM-BS;!=?u}&-gG6)2Qe$1>PK|@c0;v6f8xE&Ik~B|=&EC;PuN>?1*Ctd6#s8vY z39VTAT$8%2xi1&ZePk8EDDFI&hEnJhVQ-BRl6= z#maYeuDEO*cZaAI3|9g5RHRb7u}m4s63L;1ey}5q!^+}0`I7RCrD|ba*Ga-ysxz(G zDZ00`3q7q%-S1&zQ73T8#4^;mWYw8Q_g6iJipo;YVybQ%q&bpLDE5#En#yxEGU|Vm zjL%uJPBM1>TlHtLq>>Hs@rWrCStcIdv6PpJ^+c||&Qnc5RSORR-LocSZg>r1U?kGS zngXAF_&`cSeTN0P6uBrNdM2)C&T71Mp1mayOQ@ldOazJt)fr zX7xTtRRr|Si~c*9prR&-UwPGIM$TN5>a-}bwZ+(lxD82WY>q5;!KBy95CHvZFDKn(T{A}4 zfKJyrUGapJ?4jfkXArDkje%^ERZe8AJR>w?rTW)6s7+^37gz}|=CFFX?LVx~o z>(`NWC0no$7~8>fk5qxz`<&cdNZ&^~gTnWGNWDNF4#Gn3FY0}K-zc{ybY*DbP~u#^c3ht3_Z|` z82`-djS#)a`)afI0{49rd7tVkQ(G%yslAH0PfbM(J-fyIR`WOm=Srk?HjGq(SDPum z!AIc4q<7Ycp%%Ke%s^UAfWzP+X0nXl=ZHn1i-Ib})5yJ=bM*&#yVtGWM#KJ9ykZbX z@CxIO-~}hah(UbEAT7=yrK#T01?9OkVkYDheW*(630O3jEy4*Wk-x-4^uo~mc*XKD z5Z~rRfV$_{u6Dm@6bmQKls708gd1zw#@J!UgT)Wu=W5{D2a?dDs!N7>l^7gq)A9pr zC$?%ZD<~LPN8zLEXPTn(U=x_J>VYwcPEBaV@{JD#jTXQcGipYHBft2Fs2+rbEIevg@`@%B;Qs z$!2Z(M?VM|Og18CO{_m=IieNY5!R5a4%D@8U$`SzbPe?{jc0>gT)()yBpFkBqde0| z5lXuVyXqcTmJuoumqsR{4DZQCKmDFge!7_apCA6NKRlWI`wxHr-H>u=u4qSI7+0BC zW8FY1nR~U_-fNS*)Nt>T$vOjdPH$r_LC2~*c2vB0PN>U&O%?%g_&ilB6;TUsH7~sl zq2b$u@yt~@&Rl)utgS2Lp-t1$G*Gjcemn*nVLlJdH-=(GIuCW+R@cn%^@aLcDdK9P zO+?vXtG$tC!)YQqOJtC&8>Tic9IXke+bt3sa-A>7M5LKV{1GC;&u4qf5;^mtadVF) zxqP{oFptp!vZsYeDK0NrY1EnjN-0mw{Qon7j zfo-~q+~<+gz7b^SwrInM0HI2S`i)hr^e<=(S75n?kHWy?Z{EH3^0BA^2T;EFRO;hf zeBibPb!RLh@Dqj-&I4L}(+wTt# zbEe^_0}~Aa|I|jwC5|(0p z%6nFlThy(uL|F}+Rm1%%#fP zfB)mrsag;Vb>x(yUMx2)*~6n16R_zShjL_e;L&Xv6&Hx{f1XJ;ae;1>twnb9@? zjx~fAmrm`gdrlD(?}s0%Rq;a;zxTsG#Ow!1+Z=okH2Ief4txx`d2(WiM?7m9wdurIQDID;(Yqu1Z)T1ABXr#xUenQId1rhi8|8ms&6KAUa@jBi#idx zDVqU^DD3hq}@Y)N6smwug_^lICp-pth}7?>(?f!2r9$zu|DU~g|Blq=@hBiTvQ9)JB#u3{^xZv4=- zl=OBbIT41$%%I``JODWM8aZ}Dbgjmc$W1&p_WNOi&?fuuzcPw< zqj)#c&r!TXqj3$0**_$45q+qCP>HBuOF2^&QQB0J~hYu=mVXp0)& z=>+k;Ew^uc7bdQcOI3C|ij>&)(UVUJoZMCAyss6`2qQ8!@uKhb?Yiy}fz|*_BI$s% z+x9~X)+JsO=;!-3&uBtbRXw~Ht}_`>R>tgW8F`BSj-i)5na?#5q;s@Y-MpXGNi@fq zjA-}$#a}zyeb>*Y{}y=9uRr!8=jlxqViW4je0bY>zM0=ymZW#R6_mhC&WY%b`6|nv zV<~w;AzZB0N&MEiT{*Kkt)}QUNz1geZGzYv#O1vP{d;6`Fk)K9AKuoUt@Z|^yrY#o zbQA`kbjMWrd@F0D>@i>ODBrRC-P^;PFFwt1*R9@;BQ)zt^y6hp)HW6JW}$jc#DKpN zT=9elG4tTI^?VaEbN6dKQAQSAX>{w@ciXZ9+s{uSlaFMXw1Qk4T-8jq(eS$iGG&O1 z)C3^pR{Tf^I)YoU;-;^BMzB=&&w`}K&2}IiP}_kmngut~Dg9)p1EMBG|ItZ^;+m@; zUN;T`31X=#+ynR9LPl&EYBtR(6HTw19c&?%b&`~7U`S8p^D@Eb2T;r%c!PhJ>;%2!O8@8LE%h~2LiQ}5aUx%G z1yA35bIjDDx5g5Gjnd4}ClH%VbhlgPWhi&IIui})Qgdmy2gC0g{ZsxeZ%o5X0o^RAn;?}gP(7`QGD=`Y76FgKiiib4xu$czRlUy= zEBEQ4Hz~J$XNA{0KVS&H{F@=wMuI!G&5O*Z4Lx7sTx0(K(c~GB=zma-sgP52jwEG^ ztPlU|Z`jtNb>JEvtM*0XgfOFjizGY9augXtE7?SUO&yAf)2&mV`_h-{Wil!jS*wyt4zsFgJ( zvP!;^PkOszcl|nd;Ep@@1sXweEG#+{_yCFFaPrR(WA3R zXJ7u|?9rEhe*DEB(AnN*;2DKfSp4Daj0l?Kp68^(^SS5mXJ(ZD{Yb{ArL#ofH`4`G z%VJhYB5WGyG~ugSC6b9F@SVR~pH2VrczSk9-h!p8>zPR+($ke`ZcW)@ns7n5)L;3f zhIyY%CX-Jpx5*QPb4uPS!hi`)@BRw@ob{tm?kG#2pr?gYeD#``gxF`Almf^8#8vXq z1P5y88!6k+AKS$;BVw{3Y^sU;te{y+1ORuRUY|`LeQuR2r-kf_ zI+y#2CVYA#kjeK1bw6Yvxe06GkBd_Ln4pdghQF9?xzeA9pS61XUG=-?)Vg)@tPnVJ z6Ey^O$%T4ZeN=yBX8uAMWa!$*Ke4q0pP-knf4SyPb}!)ViQR{_v3)?UU6}^LbHdKA zE`Rv^y7B84n}qq6+_BmuTTIESLQ)@WQgT8qp|gdSS?bzpw?10KvQPS%wVtSXs$q?^ zizTUDn{i(l+1==PfN(7)%jckW1DBgZ@(~fzb+%17TN9!6@PscI{k4QGT`@>g45$SY zQq>r?Wgkv#!u19T0#^5e)!;des>5Nrpvv8@gs)ZwqiS;s7lqCWtzDj`WKFVDNf#3= zk|k9nQH3CKXhp>rj;7?AFE1R zRrl=H_BQOxP5Hjx%G79u5|GBv>(3%tpINk%bbk_lx}3&e`9 zN#AAoiJ_kkzQ&R!KON}zG&{mIS7+r|tks_TnqWahU+K9B4^!v%l})HZc9E~$gQkol zn5jzs%j=tCE3pFPj=8poU1ZfoBS%xVXxCSCli0pceu|l6Dc^9>gNj>4gU!SKhFMi) zqKEc7P+f$-n<$Kwf290wjFi7UhEo1QYK5#1+gkVDJ-ml|??8?BzCOl#Ki>QIaqoQ+ z5Xem5;#tMUX8-&B{85t2$-6LwX^0@wZ0K8A2vs@9sbv9NbBx(FvcEg)t_t9(@ztLl;- zHp`}!vn(r@ZQtGBqGXu|&;G?{-6!nk z?g+`B37+S~#?uD_oUgng0=l_o6U@=^g6&IT+Izuah8Du@c z=WHBtC&b=_hQD;i5$_(L4pWZ_ljz#4YNKqPLTBVd#{#X-2%M8sQx)5A0m|xYb}lU2 zk{dW6BX39tKfC7+X#6!X^}Fsx25;B)0`R=PAzhXcyG1*0NEgHh+oB(ER_uF^j2*4q zq%N6;d@P$}$k%IM|D47Ky{JELia5wx9~@q0sm#dSh$jmoyS)W*X#~T1vzR05WbZq4 zWoh+Z=>+@R^f9JFe{5Yb74^PF+^~^>_UnAL^;%?WW@GqvGwL z;_aOh>Y5VjmlEuh66}!@?v4`fixTdL67Gc(?t&8Te-iF|67G2t?sgLHa}w@w67Fph zGz|s%ne5of#6#{;nMV-W!z7|hNlbr|aPeYqPm+h|MiSeHB%;k;u=mK0T}SLO^e$mw zzmcti_q~7z6~Ye}J`Z&l*}1PsROgWJ{35!A?A|BDbpq+#?BN|ksy*rj8s8fvqWi~? zu7U9#KMb~e?AHT&eGKd6v3nPf=u#2=i8~QJJ07H4#}0iuYMnTG!A5uJ=cJ63}LZXGg>9ctoWo~##-%LSjoc^Sj3{^ydvF@-pJr#+!oAs$_r*l!2dU1B(S z!?0%=PVQ#>t-j4%DAGx5Yw?iFV%Ca6KXo8}aUf1-8K0f5u+;yy;pfxMD%&<*d-j{- zLM2mEZ~%;JQx*)yPcCaH*^h(e32M1C+^%+_$(AS%YhyM%3vW zQc*CjK32FomcKSuRAsDP>SBc|nLb1f(}%pe_ph9{L*2ZQ5PS@TV5drXzXvK10BZyV zUpdZB!Nt6=huF9+dBt4IF)l7<2_Fy<8ROy_Gye$U;!qjHe}x6^GtoR;sg1+i=J#(a z^O7;$K_|*w|BWajdXuWaQA$|qk}*vbp4I|=aZ ze_3KPU7MxYGi9#{E!;fNYg2;ne=lO^xjoz?fRpX>df6J?D|@MNFOUsKlZ zm3k{023P9~D02#T@1G1Y==ihSp&r9UCnlM2uTVH?BO}vE*yhzP^G_g{)P|VpqbP5U zsaj-qWr}P#6iUi8SR2Gf=N&LJGFPal6jLa{rYork)m`MLeU__s^axfs}{lLoV`dh9ysf%r8-gWv_)sh+_J2zPucWCDb zoU~tV<_Wkq>K{#-KzvEQo9fAjtH-xTcBG>64V)NWt*BWvi3=Bui?eJ~mSFKO&H93e zit4s1Q-p|PpKLHGS31TTi^^GOY1Cu%t0hV_LUciPyfwREJUh1#2$D(e>@Iz=1g*Ce zzSH81APGq|)7rJLN|}Lo-CJY3bF;qDSKfqNlptiyxY*jb`?^yw8{$s++8C$oFG00* zRxzAqwFw8z=USmK>mlG~hoVT>hy}KQw;-AVB(4_JD#q0kZbt*LNwtb-QLBD#3yN1s zmGG~&#n%fkqC)S0HbLPky-{9)>NeTrfmrLYC*`0j>1{}pXC0MSuuM=;2xyWe$fi;X z7j!|j+HC2F51Z2zE{DGQE?Nh)OskfvRsKsAdh&|}_=|n5zg>(~SV2NgFoQB+wh2%I z^$O{)q*B~c6G>8^y9cu6#sA{7!$X_$6FL2V@!2m>XdpcA_mCCQZ;}t<{*E~&{wvbJ z!`NICFUZb3l;1Gt#EsatbN(EZ>M){mQL4uuAScC*{pY5*v15)(5bk!mry@E&v~hy3*^334+e5zS`eca z9LkAl!HhbNP>xIuW*`%$1~dA}Q0`3iWGIKGdNPztQ#~2Vsi~d}<DMb9u2n5~CIfet|)~5~IM{#(zV4CYCcAvWejyif#<KO&i5dPJ6ZcG@DPVJY z0cMY)*AeXN=*Ezs^X3#Hc7DfxkAcs-ZjgifK8BW6?+O{)|1kiwqdR0|AIJb;f0xKu z(Pe;$`M$HI>OlrE0rU0e8-Wux(!&Xe)~VIAka|=_MEr#HhnAreJy2ahMKE;YVH{)T zqt-9BbGla+pe92pNfZ^?HW$c79aEfTW$c&pPP!N{-?u}lwI5R~fb0zh7@gRo6MJ-G z*T(3??tgA{V%HuVo!Fxj+cQQd_UObOo!FxjyMK&M?9qumIFMA&OfS5_E|!kD}R37G*h}d1|a8EtPq>S77pEsxX@+RHn#iLNugO|iht zZe%4B>X=1RuCZqLhf1Bsn&FL^e-t&tjcK&I%R6fC-}kn=@XB|8VyB?8`@= z{{fxtX^q}dNQK28&d!LSN$z=0DmE(n+UE5CGN-Y1jEPksufQ(Wk3P{f`w4mqm9<_I>m7gi>-m%ZQjP z2%BmeH7jVA5&^*7r`KoGN1xj|qHaCYX1r_ptl5cAEVD#gYwO&CPc(_pQv!G76V&~X ztrKXcjulyOQOX+=)UoOC7qdZ?SN?qA@ssijg&zJ z>EZFu_Tkp2+ynjl9RImM6IT3p(khWdOg|6 z-e!{8^%(8L^k|tncaw^oCJ3hpR(t)vaCdMom?30hPsUrL`v8?FKU4(bKXSNs;TjnoiLr zLr?LFWKXf|PCwh{0qC8U6TPumUu;2JnNfGCwD^0DH%+q+CJi{4=btX(3zNssKv`^GQm>Yx`|8|Q;1Vr4KXD zF*V%l%@)krreIA=n;}shP>roZq76|dm$q7(hMr@o`4W~BTGG54fiIWU0lA9jOf-)| zR7xmN09dvG=#&}zlwo^Y%jf{Qplj0AY1wQmLkX6|(sZ{F*n*5E3LWW9?f_bU*vuMo zT8&gCiE_JIr7viY%|_nT#B1FBr`HBA9)Sh-C1krOKoG=u;s7(d4}>xpi}3=tvp z7j0$vj4u`>-MJmbxmz*T(uYv)`nT>t)>GmlWbgFZ3LEpN8w#rpg6nW#dyTl#^qYww z8OP}a80;En2Yu$uV1us+J09?~s3nwJY56@&{_`^lT&FBS0oPn_%Y?J6--=(|KR{PB znOK9+RLOKUWqgwH%JcSke7<lo_;Ag>$lIJt zTe6)?NoO>ps=LV{nhApMP?0YLPKj*VBBqw_<8)1}$y$&#C2#y`yn`1WpfF;R&0DQ8<(*yhUWs&)w-B@_IVzt_uk4-Z%8sF-(};rXHZJ( zy8Q6vg7Na#7jH>Y_$xMaG}{AE#LC6yLqJLFAM~^}+3W>rw$rw4l2}n`orvW=ky^#Y ziiOU6abkvv=u3HmW`)um2Q;B2*Q^AagMcrtDK|w2IV1kcXDdx=ECH%b$*64=Kq^hw zymHSyAZ@m%{O3%C_8C#h^!Nm6_E{)>Dtg7UjaHxpwni+$i-paS&g^6Z zwQ1bJn-Y{MD1irN6AzqFs6S+Li($8HHnA73=IX)-IRLX=qU(Q^;Ev>Rd7K`QOS)LP zb9P-@o0xNtoUx{Y#H1IQqf?EwbsqpLil2~`$H(HS}dB&F(5!X=yV z{Ma9ycaB(5(hbkVw4~*}4IlABb?wAr<`7%Vrs|D3u;40SSXpmXFX{wgDS1M{LKJ~1 zW;D^yZ23Oxq7EyJ+qqPi`q*4@ozz6Y1Vv0kIFv5mJL0VY^;lmnlY?ct{DTXbE1KTbIoW~`M7`8Ts!x;+PMQD zR#kRwdDoL>`CRm^I*Cf?uEz7IqPuoM!w)0A0IwNsR@YN}7hU@6x3%USSAus4DmSX) z_7~zE0J2`PH&Bju0MusT-mW^q->TRe!Dc+~tq;7!YY_aPs_FoURv0|7{sL!{2g7tO zhytHUJgG(2YrG(08mdA(0A#CPaeOUeJEM4(X2uSeZAf)uGh%BLn-No~*gZMu21Hja zu0I~PN)Uv;0gKLuAOl(3(kKIsGSDakjWST@7-gVQ1{!4`G|E8X$75DG0Prqp&1M)2 z)*xfrKa8ERZ9Q}Kowu#D!CldUkLx>i1Ex9anirTY2zoSmbaocP)>^UA&mTwaY`EBH z42Df=Hc9VeW~Y7D?fsNW>+=lU%gbbGZ|PiUI4NCE%Ro1nzT@53W&!c>8?Y=g*}+RklyLJ&AimGOPv zx9ULqwhn3D_Sz!nv{y${8e-_OYq=gC1WbDM`XO4hflGu7>$KUPWR$lC!G4up^R^qc z7@y#(*04oC6NGugbDr_VrhX1U8^9gpn!q4g%&ZI_hz;d-!cB(b^6h-G0H&Y7-b;O7-gWy1&uP09it31%0QzG=KA7?q}YioPfym_rg3E3)jw{B;N-x#6xF>?z}r6dNO>4mBw3;$lPX9c=?K z6|S7wDYloP8J+g24j+J*KmL zP1TyuZu|jfU8CRjR9Tb;g51gegm$q(q22L%xGf3|wnw2sHYqgRE`^5MrqB*62SF&V z)wUb;YR$vB;|^!KExDNq8>wT3b4_*rj}|WA0{u@scgIvntwk8SaUF&;_S8NX4OY*H zN~G~=jWfcOjeWElv7T*P{*0#hzFPKz_=8*XizD` zL7VOa_*-uwzU|Q^0uK{yce2h#Kfb%&6*aETcVsi%%l)G9x|5&*JG4?8uY=s_*Q2v3wW8+D{M8hjPtu9Z#34|*aHR1VsCc2OsSFA zqV>eo7q!X%rs#2L5t$;uEl`|N&JLBCJ)&_At@cFRsg!OqV~sYt zZc663$l^BCoUx{nPPnmVb*uZ#xtN&}xQd%8#~L?gTrJyIOOv(+PE65xCV3pqZNW;) z+S2;2LNm<<=(u@GeH7A^;iaVjo#jhhvxQp98F%=E;TafpoAbS7HB(Mm>2QHnps0 zc0Z!M1#YC<-Qz>1+st(g+Na~hSVc{Ase>PPRAu}jq(#*Xs%4%QAO!qAj{aHiX2EK$ z8mdWIS{9XWSd9t~t*6@5b+76`*FFHx#&H!$6RWJrENv%_RZz3o%fBL+Np#@iD-LA^ zzEpS8t+BDkoLY%plmaNtl%3QfvvpKL^yV5DT zfLm6Zp#G~qQ)nU=N8krwdaQ6Z8L(&3%wmU1c5n7x$*vW>|B)54iw;z6djL;tktv#IRVtR5US=!=bDPH(d-L@SvGJVN zua9l+tB8<5!EN3f&N5TF&I8dI6-*LgXKyq&hGk4WD;v~T9(sw6G^5Qn)_3iwpO^G( zb8ln5TN5XVnkL013qO*_exbvP#Bt)wwrJvo-6gi1M2ZVuz@;b=AtwB-#`FHu3pGMO zfK4hz1m#a9+!72t&&Hb42jy)&M9$X3NwvPQCw9K^}+H2V64h3yy(NCZARM-2Kt|XEz=|>BmPH}*e zU=YUcEk8A$Fo=Pt&QD|0>~so&=t<5XJiaa-5AfWzAWk>@{)Ptwxg7JbSchd%?M9Lu zt_9zKosc#ljZZylPPix;f@~qsf_#uG!(4(in&-ly`2{Gb`Pe3xb#$L@gr8r)S^elH

u*`QkIKT zUUcSy>Hg?BGi3CFDI$K+lz}r=G=0#5={%EonvfMhT*cG{|7NETdew;w@W2NY!jrRE zcTx=&6br81j-gDcplKUQYC{U%sPPPOWDhiOoBI8F4GRt8$cazxXKZjIuCq*W5&ie^ z#3r3^3UbNVdRClop1B*Uz&pmZ+;li4Vti$NatJc8$1Yc#6nv}%X!zvU6_u0c`iFoV6P*NfHq~~u$EEAKw7R+S)PZnybh!J9Dt3lB3RJdFLjqS_HAkHIRGM_ zaX{BO6MB0T4nXKX0z7H}8V=0%Q(ObZY}nSk;nnHgf{PR8KL@-32SKB`oFr&?|8#)3 z^1q&Kz@Bg*{Fydz&J2+LZJO6Hz#ih|?}TjyY$ky5^S`%04_Ex>cpb&;DndCXu8Jb| zERM4*F`Nxao-1o<H($Gnfx}bVR-V6>IYnEtyGNV|7yt|#XKT+4JAkf0WIkw!dsd(&4X{E>*f2$Qy1D2(GjaLAMujlXW6Bvy~jW){&m7oEq6^~ zvxy|7c~{&Q1}!t_4E|inUAq0J#Pbfljv0ffg z;GovWZWs6hfsy6TT?T24#>&789K$R$F(VyVMO~(F4Eh`! zpWvK>LPu5%O^h~RQarB7ETmat>WELK+jQJ{jN`Kx0-pr~;;G9ifl)>?8Ce?*2w3R$ zj~5bR4kY`H^bE|ojf`c?US9yrW@DH+8MuI89Wwj6{xe$&i)(jy89B!5N1Kd%+fsqo zCdk@#^S6#0cNke3---uz*0-*Qi{Mtj1$B5bIq3!{CrIA%y#MXf1$3JF$&{0A^qF}p zQ%NKon~0W~NVUeyyBuRZY03nD)5B_~ZLU6vm9rYIOg)~|N^PqME|o8D&6dYZMrBL^ zIHFiL`C#V2Ot5#Q1K1==A3B9Ors)INEW}+}*-cJnb+A&ky_J{97cwonC>ZhGA<1na z@eTJ#sgeTpM>x1U)e!L9YA`=z0mL=otEfM0l(_l2+*pJJrMOI`!j^;-(v(A^N_BH^ zAsIO;9yc0%MkM^_I@{e)IgCj0a%<;`m@q#sv4h_Z^mVCzLBiGFt)v{fYg!||KCes^ zApQZ6ZU%J?I^FOzmuVh1Te`shMVdN5<%du)!kk>$>f#K=K^2mEK?jqdi~rGR%P3W)-8fh#Iik`yhALymT1iGC%s(wSa} z)Fi1Sv{3D>`4`AK0m<4^v=r~uHYs)?`Fx>2DOM1dn~76j3_F(q!7%?`!i&cZ;7ck!&bNsa)+(U4Yen`&D0H)Ly@^Za#OsK?q?Uj|D0 z)M}lNB$<(TO25&=%o7)TH_JO5BU^D~1gi6};*+CT zLkEhV{PQ5NGTR5H*P97aP9NZ5&b z=JkmCOI!LQQDQ`Y2bGm@ab}@aVLy~l6^F@Y9A#S$flwGW2mAt##P2Dv$0kO_NvVTl z+e*8{c>1$A<_gGFggx^HXpV}+~n;n>*Zd+?xn{y$`gP!YkdCMO38_{K1 zR_`<(a_Y?@k>ohkQpGV@%JIR~k%g+EYa1 z%Jlzwc>Y3Goyeju^Nvj+4&)R1`R($2b3WEI;rI9N7(tl;fgjlS-=&2Gw_CmLkLR74 zt|&M4vB#>Tx?cXTYBMwL3SEmHSjDWx+NH{437PByQS zx6Qpuzl*-lvzmxMb=P31S1e1n{xC4yWYkg}sd>-`GXqP1B-4N-Gx%*FEvcu3mw&j1ZB)Bk4v9@-gTo9y_DN`YZKRSf1beEf zng=7e85%+NI5S$A*&YY>?Ib_ulmUXW4#~6y9jRZ}(X{uOuCr*JW$tHOVl6gIA8aI4 zfxC>qN_3l^hD1@1pJI9i0$J;f==Y) z&>3jedV``JM|u3|tbOoM^RRa@An` zP}&e%b_R674U%U3IP{#i2KQcAC}Osl=Bxe%x=kv1gP2U=XrRz3goKfRua9GW85c>c zZ85(Lgvp!hRt6NoVo`(E)8$Gnw^;+V6gIraE|{tTNO3-X1v|9vE)dt#@|@4-UM7!w z+p8VEl#uLb5{msd>Zd;#n3WvWnbhOP0>W9HpNGfSVPW57Krg|uO>D*~MarYH+%018 zIvZMcji7#Sb`ZZ^?m>db6tYQxSf<52UaeP2F$6+nvS|yZ z`jS?2jRi;AD7FM@s@tDs`vJ*FwA|&Ji;4tJ3Ml;=zRXCI<&;#`EQ1_IeNWa9bYGpdryX37{(ho2VK~L zt$-616+R5nsT=04WS7EDq@>V`C-%FQ)YqM4*CDz5g7eS-*x}vGAM>A$NL|l(8InJ& zCGM^asDQ#34$Wd3f_u0`f)%FpfCcd$8dUN9>G5&h6CD=%qh4b6tB86T&Wp4<4Lj@K z6hbf6^2rBU54AT8g}!%}3!VnU<1DCS@&VrgD`o=h*hXEP3Rciikfc!6&Dh#TpFKud z*6v43o$ck-jM8eRBm@U?yw3L0>mX;FPPUCpJ*S#{g1|21kC57P5XIwaGCxUSuMvhWYgtBkoUwyFfJwOTB<2 zl+qY$L9C|5%;m5X6K&Aoi(2htH7Z!h>Xx))2tx=zTf1lAgXWG#cGjtgmdMxF>5RV# zp>cy)$r{H$@*9hh=P<&xfM8`toyW{Q?RyUIw5Tb9Z>h z>GN+iO@{_92Z2OyNi{9Bbcqo#z4)X*APxRgXK?4Eq-_UbREq#T2&U$hlDD6{Rvo$BgbCmg5a4~j?nK$y)zbTxj@|m2j9kqA zeY`pU>3Hu%8h>XC?XwyQKhV2Wz)1voBASrrE0z4Lb&c>B!BN*Bo@@XRS>$?)D1{ zQ^XJK2oQe3TXkY@fBqrIGJ^St_sF8`GqF(BxVGZGqit7owVmu7`7&r8#%(vWJI|KC zy63!8h^C&_SPT;~Lfu(ujQlx0;k&R+lf_1lnF2c&8YGS44HxYVN zC#{W1D1%CWf{ifj4YyJyPErw?)L&}}%;_bBl?IW4bHHJgu1hNXTnARc*~Kjkx7WT> z$4@wL>Io)&sut$C=`8hh_H#=dBSCT!aD?Vx9=B3+!}ArIM&4+&t&@&!Cw?;j5_^e8 z;G*dqj6fu-0@-n6e+*KW8wbl!maj(IgifqM`&Ar2&s?V9rW`a01wunSAUS>qUSp-{ zR5hg$5f&cGsPt-oXunee6P;YZ?rWAjKs-E@BKUeOjS*a|5RM)brRK8<%LJB85M8Tk zG)N`*du%eRGMPCAt;8{c)?5BBw;_#IH$I+1bo=9b(m!1^QN$qN)}vSCCRP&Q8K#-)_f-X0G$fiyd$&?Uw&*>mW{H|Gawv>=h7!(UlXU@A z4sZ=IP*ow-sVPgXrf3Q`WHgzD>C*EDp5!~#T%a#t&kqhGjAe#zE~4z$@e(u$Tte;w;3Nd<>?1fr!2L>fmzydt+m`57;NP5aW*v1sP@ zNOm!j$%|h&((DRDg!L)w3Q}OrPCNM?*76UVfYXoa3>ptF7a!ja%e_$MbzFu0$Kvtg z%p_dAbBYKAb0TgjMR?`9?Mo){3II05n~Flpr39QNS$C%`(Ss2}l&v@)2fv4xOIVm_ z{21CK0=C5^#!-5w{nh}-fJoASuH@6~A¥2sh6(3}dSFr@qmH^Zbxbm7SOznmVtD zrJqz}2{Dzj$jY&klv7#y2-YlH)|V`B?8au`YKtj0eEiq6c{>|#ueNV9Wp%|)TLH3v z&E@+`R#uf=gk=I6+=LjGzn1iHcGL?0Z)v|cus!Pz6>_2tdzVrd#Sxs@V{_*C2UJ}> z*#!ZQ0yF+0K!WUPi&Wu!z}L1AC;1+|M8)4*(6V?g>q^7nwxT%EtWAzklv82qUSVmT zkzdf!G}y>3N--?-H+u2gIN6Kp*=`kNMel>ectVlt1qai#5=DZ=w>Ie< z=x~4J3+X?iG_zKIM?bqCb(V^XfcaSmVwtfHV11?(mydHb8RGTpJztp6 zO^J{M9`d@BaOcPRYl`58792dHj*xvUhxuh4b++8{Zn;wsLG!F+#*NN-$D27o&IKyu zl>1N5HU_r}CemSmtu5{Kn0uur{7i7_YdjTq&R@M_tQio}!s~Dwd&t z{A1V4N|=){*8n|@?y&wOF}FV@@cs56AV0&3M>;)!@;3FLU+d=%J)y`{=6SnL4QUr! z2rqj{(XCb&X?+4;^>=i%-@>CT!=hxtbM6&aT7}%7@FS7OR9qt8+s8}!3CAYQ;zH|{ z`$#@G_5v1sl@gRih=utqFdr*`gn9<&BUz73l2bYi#&GSFuMAH;Z522=Mc=T1Y1}xF zEdbZ&O^mg11gXy5x~XalW3Ym>?{s2EP_^uWi$jbCN5yb{fNxOi)cfX2 z#IJ3YttMhOK__J*iY0ryXMh;23Qf&foCRBoOSF_Ugb=Z4#=VBKp6IGXcQ0M#3YXZK z&e!auiK`SV{U^+?NpPWhiNv&DgPlz89BVJ|`dn$Mbr%>F;wmO>rSBub9iOwsP}1=- zSwW*cQgpy+9pTx^3de)&YrD-IZpQcc{AibBp( zfk=nOk0D$+E{;L@J;0DlFM18{+YoYU?#}V?yq8xpy_bsg`ons5`b|yZH`8fX{lh1iD0Dw$ z`u&6D_gLbdgz}e2`aMME9Z5;A#DmWcE5VSPNa;taCO<8eXx6nA`5WOfW|GemF2U|X zeMhfNWE@hGcj>!3i+6F0{g1UM7Gu!_(@Ww$O=#jY-iO_W(N9kyEL}93m|7)$vQGNX z>&9fF0L(Ws!+K*&LRK<`UZkbyV)iD7h_!jB8VO|K{JzvB0TDrgyi^uit1f{*IV!Sq zuYVov)kO}cg^%x9AGg-j<*iszS$+yrX+OZa{>TbrW!Z%gia~nFbfu#cWTRt+NpGo$ zxC$;JOmu^icNw7kLHHa$PbfA|Z!{l@yN*6oLadX-t~(S1smeKLzY+xJ`gkj-l1ne? z__Fz47fVW15?&98#7&n%>E~FIvV3z)yxtD{*x3fIzd0E(F!#n`c9HRT7dbPe(by?$R}RW zQKtL5;pQa@sl&)SZsD(^|C~d6AJQxjt@PEYRdg~g{@_@9m!Ctrox|8>w$~pj{$K%z z+uq@T(|92MVV^eyf@)^_4#7Lom!|cdDJ_h%7o3X+map#p#<2Gfe$F?z&by;4tckl*?yE=3l*BDZ zJFK2&EWGYv-`ex5kky*A=-AYPz^dVC=GG>P$ZF~yj3lDXE=OzO+%Wvny*HfUvq(l~ z(F~rrqfAC!P29f{mWo<^Z|BoM?=~}F2K(`j0&18rSH6I}6%#!ZS>vxEFbmJGwNG?Fiekv6h_)uolAZI~g;bv7wklB@ z+31Mdy zZ!4h~gELNA*b-H~4Xiv7)|ZE3{gVcSY+gPm7jExEB9=}LWa_3h;C}S4nWdxe7qN(1 zSe^mGHJ^#-jI-p?w2a~CsS?=Ptb2?%jH$v;pYgcU58~jzGc#iY--Gdx@t9&cgPGBy zD`m&S>C-}lsr-gwYo5ZSf)}DGbzoQtSC@8;|$LG|;LmflzOVKu%TL$HjHwcFb=UZYhyiT^#~~dY#ox()QZcQz9XoK>Tlg%Pw0}0Ia1T~c=hZ; z13pRcqNdswq&TXm<>R2wfN~c|e{}yZOU^bGV4B8!C@rK0Ldr2}%|f)@{9qWxV|+!~ zI#?QBu|4|$wVT|_s*;hIOItT{mx0UgVkLCZTz>A4%Oe4=<9!GVU}f->fRAo;y4aoG z%!d_>2@%@=Fx|09pQ$P9;*2ZJ;H+Ndls(GYS1kuEv##gN$M8+*=5H<)PV0L19%vT! zfwS{_^V*Lb3P!z;(sY2DCJpbo78kq*eqRN49lkKSdI4e~*a4UcZIO_4+v`4C(p~VM z1$+eglK1V|5b*U#=!hP{K=G&kE$*Lg^efU{T$wGmRdzLi(3PO|HXj|Wgep<5C{-9F z1d3H_F_8MzR|3t|i*@$)Nw?pc_N3=LWg<7eE=^CugR|v&SFVOu0@-KUk74{9yP#}o z98!bqULwhQ>D_5|TO@Ks`&I`cwgBVEZmy2~NDAKhi{Zvq3Lzfdr2#X;p-!hD&*%JJ zKUbH+03nR9%Q8cKDP2_lb`MXq^LYH0_jGh0H~{5c@$DX2i7D6OBAl~D2T7J5I66m< zq{sLI0eh0}fo+#RA!mtKG_Mm7$v@GJ_vpYfP~x4Y;}A=hT`_^V&PJNv zLSKxlTl)fPV0sSSZ}!6b0o(6iPEO4Lv&$5%(rA$YdIU7dQI%X7?GO4Bi^=9D8V) zdIfyKWf?r0@_r3n8Hp-f-m-M9p0>k1pV;{d((t!NOWG&svmx-?6;EWH`QlIV>6*Bx zQm;vMj{=Dxjr1I5BhVBkjJvOUTPQQ9&U(kIirXR(9;5zU1%$B`u~AI6w6klIc6JhSXRr!6L>kt2>l*~yHbufHjS?jGQ) zow~0g%Q9m{)TheHElxaeYEG2o@8MMttEp$h*V2DsV-=Eb$Ynuq(HF`xJJgOrwjJe5 zZ4+@E-mPDiYr@BLiz_^PiG3)4#7j~)D(0&va1t1X;1~tr7z9i;D2dinocG20zAtr6 z+^hfmDUd7sb#x|x_sJUK_xJw)cl?9@LGBWNwUNBc7rsmt=3(B=J!CJiG+4dcU=nl) zL7WL|4x*T%&%nBHJv2KBZ#Kz=W3$6X?-0E}W`%)JR*ANcE zZY`h|jN(!DNprrB`{9gXxF6D5G$gA79?#%*#T0v2IQ0F%Tfoun_TrD7%R4H0A7~4; zGnS(ltDC`5p@Vxc%Qel=QS)U8;vOd|F{c{`mLT0(m3!8;dze#1XiDD>vrnTJaq(~2 zq!$tT-q(Sq*?W=9OYA)te1ll5@d(X$W169$ny6T{Gz_kxs`^Zikroc^4+nbdKZaWb z<>mLE*tz}7iXAcrwJ%q7wdUzo(?aEw7vFS5!X7zBN1(G9UTBt9qOH*rq1MYOGKArx zLFkStG;oK*-_A0${umj;S=B5hL3=}Z9xQrPX2v}Zrey9n24nJ|P@H~s52)B^W5;}K zWe3C@-EZKtxzvVeG5t|%`NbxdMn3&k{3hydH#M?~OEufl)+3PXWuS<=Xae_Rnw_2$ zB%9DY!juk~bW6p3-6|MXiZ78gB;WEOnMG{(nfpFV<-7AuDd>jBKm3dwW9BSq`vR1c zBp#{E4iOSLk3PwAv(+kL=%Mc5%skZM{Q?H1^nt#YMyOoJWyvG)QnJQxGQqji?F`}GFNhDxq1E=V>yQh{YnZ( zCWA^hk*9z0)w|_ys&RpQ|C?QpeZcz+=bW_^MSfRKn(UM5UkvfxzlD9I3%o;7_VpcX zRxMlEs`MwlfU?slEs(K{h;NexdVR0*^F_w}HnfURdv?j*-mvOlzu#6E0MYZWlfN5L zQ)%a9>ccIYv?Vdc&A0C+#C%d?{mD~Q>otVpc_n{i9y^&~$rd1lc~fAKvyII&Fct1v zjdm^>fQ_Y=mQ?PxW!Aw2mSqxlRr#&h;GH{^?4KV_$IyXU!XcituQSgqJ{aXd#tz(4 zQBDNl^APCOV&uY3Eb_{zBym1a9I>wBQ6H8A>bM9OR;n(izU{`-62vAyu}F_t;8=gb zZ?3Jix(>8Ay~rohb4hy}+*}5-(X^{F&PSG+MAjAt)@RzSf>{g2`{qzVQ%lS4LIzI@ zP6UXP6>{Ha9?3i_WMEDOo4R!=WpHNJd7Q!M)xVZ+|A9+J9!*KXeOk+xO4^5FRx9t{ z?Qo6+w@D?(pG@$k-U-#~r~}6&l}*KNydr-~(UtPkM9c>FvEWjZE()rt?{%U3G72vGFqEF3i(68s-ReHo@ljq~3Lnk(!;EM? z2?b+<0=a@(cYQ4RW2dh=Su`y;uf5{8v%3WzTAM|=-tkxMZg47~6|~xpAS%Jh}*$2>C~x*{|IlWYAg zMtRGHbX9v;(;z{byNc65Qc^Nns7Ze^sn?K!gY(8ce-Xi|wCvweSTP!D^oz~bU;16i z`=o8yadq<~z**FH1HyrHzu+&DxoU1Bu_hkj7ipbhRwNngGivVw39F@GRdJEXVGI!} zZH*h1|9<_+z&OU@UUcFG-wyP>ip2Sl(O%2qxf`eqwAVz$2))ZD98I82H0=rKW`W&o zu2iU3z1$VsrDGUxZ87WLf!6(CKdBy3t7VvEM@QZ z0}mFa28X5)Rnntee@7-;_#-!GM0^Mjos|}B3cQwKOY(1#F4~Q5kh~p3&F0^@ zzcMO*2yQW`Pk4+vf%ip4DEt&R8a3#&2NxMtR&vV8Y~RC4IxfkB5q93w_6-f~{$j-1`0vxbQlC^&NEmnR(?u%&E z{>)jRftA;r-X#^d4y&T|Wa5ObN{U~)dJb;Rxm6LR_IPJ!AjsEi=Sr`%zJrFQ>#vQ} z-yqicxntLB6z|TV`P%LnvM)U>!!l7y9|mDjDZ}z1dZC4!U9+Dj6ANk{I~Z$cGiyRy z4IcEMU$OR`g-W4=^pEZ$zochx=M&o4W# zsP_qF5F!{`){_x4w+!CJ&x>|($~O2b?QL!D+V!-+T2#$A+V5uhSQ@XtBBQW{Xows& z92WZ2Zwr#sF%^Lw#NBDc=|TxOt2nrW6Zy~|4R-_`b4@+Z7(t&yT{1H3IxcvX(Cb+f zTD1l%Ye*rN=4*;7F?(Pl)V5vNH5b}EQ$_My2j#HLlHQ-y{P=SVrPWG9UngjpTj#h5 zF`P?S$*N!G0zL;aVAs4_n^mMs3d4s7;|H)-VrGNta4_?(qcY<-$+s+`;cM-x-gL`^ zv;-DerN$_UWs6+K8Sy|Gra;{hU9+$hXqk-x)3f+RlCX1#rKtevd- zJ{d8gb}{A#X(KsG&(k#n9;53D5C-t@x~?Ai*jzWTx~{ARI9xX>Vj8C4qZ_8=)-VA* zbM{w_EUp_H*v4r&4O6jp(f?+-{;&C*1GL_P0-J{mfbMIi|2YB7k2Ul5KAc%MEwlRH zz<-lx*GWZKTXX_g~IL(7i)XyiDb-Rb$|#h}}?mADMuhGywsR1$|BiR0S7`I-K2tWp)tn+d!^8eE4E z5Vy~GvAqH}!;I7{I1lEZEG-uis{VQ&8}D^$p%^D(oEDWyjF5dq4YU6DP6K#G`} zR@>xI3r0Y;yW@X?c=WrmHzh@;X&rfteS@+%SS!kDRCNaCa24i z*LEQ*Q3F6ixpi|vWK$66ROH^20pLnsIrbNhx-^xYIb|o_n_X>_q8Uo@dh%T9(YFH%?^_OaYY3Z< z?PPk|;9+6RAWwN2n2p)K#EsdnG>9f#bLGx_{#=)X>=g2|r_RbIY-u=K(Jix9=b>`> zJmX}W0>bG#cW2&2`VBBd?dG$5Lz$HfFs^4bDf+TBM=+NyE(MdiIF^pFbSU?W2d9M- z+FBJwutjqOG;fKWzOf7I3y%D=;hj2{4^h`H4@BB zWR0`t*mmvJQlq8H16ils9mAA9exB4kD6z?JIjK**!{QVW&}5$_)5&yq_uEZ7_q^lv z&%eTVOnF^eTKxFX@EGf?OxtcfyR$<{=PL(Ct7PV!WW&ZUdV#b($KT`_1#Mwf8|SrI zW}J+OSFu$30?B4%6^Eg1V&2&C<6_#OtYZ1YdklBFE&?Ov1e9-LzMa#1Sm6UjUqjWj zNmM*Ca_}>0pD67;%u;rSr^ZiN5Ad6QR{r9bVh9Xk$GmOGlR_S51Nr-M-V$;BQ)dU5 z#zo#{2igZ5osrwyu|4g>8lA>H7Gt0nxQ4Awzou;2@&Mrr{`9s#&Lgu6Nti*;i(~p)78x+`vzph#7#YWsT$0wjt{Cbe#UJE((Qho6L^ zc$qSFon3RRZ4)}VyjiJm)Ej*GLJ*ZRyzT$M%b-AzZB!*h2|&Lo0a1j7KA=NAve2T5lZgh6)LI(Ygk}gWgn30v^7Pf5 zhxcj!h3!3D(9MvvX}P8^vF2XY8@V|~3?id2ht$P({s5GDzFKV7K#lsi2rWo%msB0^ z+=K9rNFoCGKjuI!u$DVw*^FA*iFrggapgAox%K!};g&ItV4C5eF@u8gMbGGGWx%+S-K&8C-g>w{s6~B;aA%$_*9L1oBs|(e;nCV4?eJ)!vE!l;-F0S@*;!TpjUs|kYvuM9%tidEBSpr$4&>Nc;YLm=b&wPVIcz%P7HHLFf_8Zw;wMuF6&y+aJxngybzlXU2a#qXgN8)11Yj(%_&gN^eb0*)a6Pt<=e}REMjSxqkQtfN<@|rChAwcAsMj zm#mqLpuL5qvg_dl`Ho%R4p(B+uly0S{ssc6h!(2Oz3vw9gdUgR{TD_HiuoLSyG%=C z=<`wHeyQ=kJw#;^G2t(hj02IFYWI$%)JBAQPaLk|B`N(SP&kSCgsg+P#fhN`RTtft z@mNV5LK=@lT+q_)6&m**H@+eDNNhBj78lIt8)9%fxAHfq*Ra|b!y zLGL26s2{+O20(A?AW{hxrg51i=3~iogf1tifCv12Oc=6sDm@2}0#jTB4uVc(Cxp)b z^*(3noV4o5cuA^nU>!O~vA(ee1QekNRo-FVW3~+B^y&WoT9;F%#vaZ8Uf1MSI`cr_ zm;L4aH|6@(`*EiW2Pe zSGZcZz4v{Z9O~mwu{U75?Z3H~AYBl$6}x)@`6}a}?j(wG!6#ZhlnX#hSMF-F$F7Xj z&JI1?j?0D!;sKUXuH5fq2%e7&_mR91bB{g=`Ct{oVf3?mlG@c_-NF#|u+E2~5YzO7 z2`)0#2t3?^;)$_0bL3!mkO2!?5jRf8wtM3cL?TGRxOT@2%G#55S5MT(=o zW(!OO_QTw4b48@(N@LD+@?Fy!20Pn;T5>=H(8stzBt4&N)TY#LJ0hBTIx%%v|zB^t4MZ@G{VY>(^tP zt85y2f>)I_`HuVv@@grXlOB)AP{_Hn|4*HHFjF?mAH7cI&GG(kDYM#U zyQszQSERL%Pd~c6SlCd}-!`xHeJC32NO*MRp2fp&a;}886}D#w*T~o24RiBTGEhyU zEC#_295e5F&6|E9iNNhTICVLzBG6V?PGG^2V{UkEai!HRi3o~D?XFHF2~9zJI{PqD z;*r_Bw*{A@Kzd?`{t+h4edN6CuO241H!)73y+yjv7LJmxcHc#m2irT3t4Xx5<5v3lkdu)s}dmS=f_+0EL+zBl5lEW4*w{UOB{i?zZI6)HkgphUXz_N8cpvv49 zw856EeZ(B#2)>T8N7Z_d2dRqULG$ibDh*f%_&h1yy|8_OVnq9LWcp)_gZwuD;?KIx%3$!t-qZz9;?TqOqB8m0d#;_r5Zn(OW5G(RT z_&ZVBC0CI)sDaqj0w!%WU!TpzlfF*tg2MDWPzzf3h7dy#ZbpX|G09W;(x<*Bl#}T4 z)|ShBf<@bLh>UYO+*1TnBu$;s;?p#rJLkCCiHltuelv18@Em7qUF?~<40*)*>n4{z zh~h4~4yu|)mW}=jQwtxd3Oy3B2p_32cVA6z#X%-q6`a+7$0uHpW-me>@w)Q0fyB*X zrF((nSL5K@ajZ4>bKk?(){p5kwf4`eM$2=Q#lzkOm&I0Z6}O+@4QTs`v( z0Nu^J(}?bGB@88c8xc%V^?nMO9Y5}{a%%ObHBnTe&66|!NF+Gjd3^sm&=F(41Nd}8 z>9yxE&_tpBjR-?la$>-DtHF$+P7&TBGv9!B2D&2rH59tyOK4@$Rqo=@q+P)~1WlHe z2NSba7$X`+DRORYRuq?0Mk6Z%f@3Ia8luV9bggTHru#hpch=gm1qu+nV*BUy^%Xq_Z+ z@YBwD>G1KD&KQ%?GZ>p@Kvi%i5I>t(!hC)yDd_xkP`6n9uB1D>C|WTz1v#ku2GVik>$ zi8`zKbcb$|XmHTPY=+IK7D{TCt1`^kM0_I;aCc!+i&T$iSiu$cRo6o(eR(ct1)kvx z6nbL#L}5Nv1H?-OEz~%uZ_$A?OAnM83h(bpg-+Z zVo4vyB{qNTKBCOMcixgb-4?kpKi?$=4>7a+zE0RNTnNae!vKC&QVp<1o$a(Yl4Tlv z=Lay5vkK7=qbryZYId%}Y^?z$LX}sRAU~PBV8ZBVUKHpMJSKmWbY36+9v%6dY}e?j z`}J`2r${b)V`afrZQ|d3v^gXA;6ko0)PlI1z>3Rz-3Nw`WOpIro1IL>R8KU}6jTCA zC?1u^Hj4a%O98UwitNWSQ{FGdJVBBSB8*#{0WXo3Y4cV!76*e_rM-|DQ*luP3${R; zk+#a3;Hjhgq43)l3tBh^`!BqWZt{Uz&TFxtAw^<9#rV`-P5i^j3 zOmS)+BbxL^_n#sutDoQ?D>yB_Ao)vBoVC9ehn>IN-Cx^o>A&9c!omdhM)%#H_!T^i zgKU2=T^nG^6S0zlmqKANIxc$vFt$|gE2$^B^Z?o+^^-U*d4{^e{ngyG0el+~C{HmW z*Y-pKm0lm?RlUR%3+%@j@gH_Q&8PW|)Kzo&{OzeSnYwcbt8u(iZa%Q2QcfHjc&$I6 z*EXJTsu?nLva>J6v1S4R{O`KU!yvS7kgAzK^A5d=Wa($0v%Hol7bdYOQwlL@{i{ov zS%yg?CmM6m5~K=A9;Pf3F__l2`E39Ruk8eqy`(fX&@z*VjxPZk{U;CL&c=ybcOsqe zjlSWMicar}h3Dyr68;BLLl!5;t2?n=pDzQN( z6s6Ww{!p?;TNhc3*8j?!aVw5w)FiAT zR{VKBv0)qgP!ZuB)2K!X9)}Z1X7lHUPin zZ?@pTS1lAA4}o84{wUGnywTO>uer$kSjD6SO8S2>1SlhU0+Tm4mb^d{F~%*R8aBL{ z_y)O%iuK&HoXt2hN`-4X)s+mGCUSgrVxy^r!pi+g%kkhh7($NyGHE;r%{X&ptM$p_ zM<+()1SB~95_uFHdoCBv zNLuIy#z*Qg%t3<-o?2iPlH4t{BKqhx71&BRiG9gkRkb)r?K$MXnU*z%nn>a?KN@~P zIrF%S0}3?9h1$ZM)t?A$G%aJKP%2xOr)tfZ4 z!vp|qNo3tBiE{J`>!2GQA;p1K`br##Bx~o7b7(E26f;pl4w#B=HZ+lyDK5e8n4K~9 z5phlj@?i9^G`P2tvtgi$@Vp|Gz1L7Ol+ z(MImUNIZ#=KiP-M_c`a3)_sBuwsDcNkzz?pRVm3jwmm|-RiWnF^sA4GxrsT2yE^%W z0}BHObfEG{(1c5GE9>3~j-X_^M{5wv-sYApp4{mxVZ{;!2DXsa1QXB+c+>&94s7?@ zR379oBAkVJg!tV&yh5VnBY)rlVKB{3k+w29-v6I;fbz3)qGMuM5dF52DXCFG-W?Y= zs0=eJykP`$J4K0fAWIYLwEgfJ*pCIV|3wG*^Of>LXbp($o4$=fDQYNp*hQp1=T&)X zaB!&apgevxJkw9dZ!G8B^aRdty^?XA<%nllGwVMUC#BaF-tj)fn0Q@#c9OD2T#1Q- zzyV8H);#-t!JGh*t;(Y0E!K}fdsNu+THfvY*yr>#45`79W10e4($S>W<5A5<&tQGl zr~Fu5*xHu*%><5Z=VIurNT>ac9Iz{lCS_&MjQNehAmN>oY%F?IMOru-Wbi^@kAR-I(FDqs2bTCDVQSu5i3`s^Hb>45TxsQfE(zu5=S@ z_D-3SP#)JG>e73ZpfF^%3j-}--BGBlmjY>!DGB}FE{4#{GbgE*pGcSm9S7OfR2j6h>`6@jpb zLs-nlEm1-GT>Suh)VHN}W=3*gQicC@%tLk>?R{0AXN2C;k&{mFev9b$_k-|wRaQI# z#?kfl>S1BXje(f!hnXrNo;LX2l;UQ-%}4NzHxf0Co5m3bJH+%T&!r4aL)CLvHi`EA{vo0{IlkxyyK!?amDu(@s` z^+ckta^(z1IO8}T1?2WUFie_dKGX)Sb zX6M~59|S7E?Z}O#_!*_yN-p74DAw{^{qWA7c(|!eCm5JhCzs7rpt=Tvws~$u;ZAM| zq+N}KJxT$;MRNG)73x=o4`B6%;%@{3v$ho{$waj~*)thz-SZ51tjwzv=|c5>INP$o zDms-%=KB8AS4x?K7lWd21Ckai`$f2RWRimFi;E$%T<$9;cHgi~* z;jNMwYk8+A%U?bw-t;K*Z^zO;CeHR!`VRPaeP*@Abgd|Vm-HXNY^9iGGaP&!k=YQ0OmezkP*GBYl(an=1} z3T%#g+gdeD`N7uW=TosaH)ElpB-TY*D3m;QFNac2-J%(L^(nlHa798eQ)$i7!wk~g z&dhg`-|E#&sM`1fi|-dNX9cl?nJUID@54Fb2?HO2RETlD&gLkbng71@G4nR0$Tw=R z?n*~0*hI%l^M}odglCVcb*v+pQlAk9DO{I~Sqix^2#bvvvN^-G3d(D~n5Gpu9|jZK z#W~pBh&jPQ$Jy`KH!_Qbg`oC_tem!&adxn`tyuIL%l824lFm3Pja89?l%Ue(L zy`T{ejUL_cEwxl;uNr^&vHGorL^v)s50$Pn;nx4r>BdvTdosPi{!VV=q_fgx^2U#K)cAyr z;z0g2<|lG{oQew8xtLiaol$%5wIzII&f3R6WvXK5+^0dLHkcL34Gx16kfNGmqwr-- zUSlW+(aG#hbjDktMwPJAA?8di6~Qq`^i%s_qe2^`Aks(}b$#4K$=kH}yF5VL)@an) z4G&0CYQ4-~l8rdFDa%cFscMl|UUmUWOqs$Ll$slvNOZU~Bjbx|gNXc0ljz8DsON=E zbzIHl>fHNv!S%(cqcuqr&D1{lRV=uqHoYC5_vyVGq#6bUM^z83_n*{u?vcytEx`)o z_Hk4Omn)ru5=^MpKZP1=Wl)~UnggyF<3V11B@j}6(gZz(8vhh(=(k11hn6G@W>yV6 z{{{`Jmy;>VI?+d#UkA|hP5M|LL1JhV6@#JMM}?ztjbQ0{Fr#R(msH0_BQ4prEc@CB zw;^+K!N9gbQ;OpVdWR0yyqrAhT$)< zLT<{>VRZ-TuY6kAw`PI8krXo|Z% z<{^z^rgI<%T7)nD`TeF=>3cfU!ap)|9qM4$~L{y$0 zuPA$RNTZpsneF7%7~SeHcZpzw(E63DQx;i9Ig|69Ye@)kxGPQ#q73;>8f->kl-TMy z_)H1m7+^$3sUBaC`A$e#6Q>nxlScBfW^I?zJ*`UA)Z=r(whdg!H9oskIjw5h|3tO0 z7tTrOiQ~D$R51GFq-`1ugBU#X(pdBueD}7KYkP^z$`1M&QYa|1nr-$Q+4iFI^0!Wy zh4~C@I5h!aq8FsOAA}^8Q4j@lHGIV{RjQ{UtT4uBr!g5<4kd4u+e=JCp?&RdWDyDp z9^c44PK?aEw~2%#;N5x7@|jD~WlD70+K<8!Wc~?HT&T zmYQjKIHuBl`&;G<>D!-sLEk5X0GLz7!dU9Lm3G3{Z^amO$+dbs>{E#HMx6%PYjBK} zF3qmdIe24$)4B~N6}IdP0qKlxw1%vkAnO|PqEyNRZc0JYF%?rhMgs0>f2{QNZ*IIA z{-NLU4yVdGuV|_N_t4MQK~`DE(+{zv^?{n+=Olf9{(F=Uh?LIVpTA`J*X-k3#^F>% z)URO=z=nzqR(L>ATMVGy8V%uid}Cx(>1fw5v=azXqpGhZAL0iY6ho z6QtMVC4qOAJ%t(X9`%IU2s>?G``W4n(=I3Dig@ZX>+0*y`#Kzi0>gZ>{PxrhQAYtibId1_gqgj{ zgx|SzO2x2Fo*gk4X#=K}4rH1k%52L7@bj4S6cS1p;@j zLuK3#(mh+GjFIV})zU$=xz^Dw1n}0d>Y7-~O%tJ{OCX0IXffW@Xc;%@ZuGeaKs}qE zJVGkR#Lvj2!PA42%KULORy)!u#xyh3vV+uVy5*nY^gF3GE{Yhj3&~vMi@jNJ0Ny*n z@0i{)Jz-|^#w*tnLzk>r!EH)mSs1cX5Gu8`74sDFFk=@{p=?M*nLx*6&kD<%R9@{G^a@ z^_-A20*~FPqIVqwRvgxeM&Lu3m=UG!K7cWW#hW&0BBf~Rbtt=@{1jnk+bZ0;_2_420~ff@yKshR4W|A33BO&D8t*PC=Bi%HcSon zt};FnaI`qYB>WOhF{$Qq<}(_G<}g`ZraL{XZ`j~SZ(VRO`5)Ne{_+wiCt=wTM!pKy z%|@uDqnq73A&DV1l)I=0TJso9`%1kylVFXO3~u!%Q3e?Fpcm=|nhw!u@O?q1#B0DRm;=0 zRKdkHz<84QguB;P1$oY;lx1b-<_j?$u(cmY5d~Jd+8PIE9~j4`sz)Z#Z1H4RgUn3z zNR^^OzLJ@$WbV3hK4|Ck5%nZa*L@9zqG>?C9*5{yg0dSZB$mY`L?l5~>>%l15op=D z!6HFJh08e+WP;dJ>OJ{C+`8QTs z&Gn4jxQRaml1vQv9y7p_(>wZEA2UpJs2$YLfVk)xRz@DD#ZKSaRq%yLeqPD;Zq0pA zCjpg$nBC=BqBfbetSoVG!=|1G@QVcG`?gs;6%6@*U<05-LD+~=yJw;kc{&c-a7i@tncik&gr%DYHU$C~ zh`{vn&x5BZS%W{V$;q6m`ll9ts^c5;$vhFM)B8yqzATYfECLHAmL4jQ%m$EENKJH^H~I;97n;4Sq_hA`I9PKeKy zZjQg9>YNV$g{Yhe4}(P7gqKoOqM+R55;M*%R47vo9j1Am+bteo6?gbe8w{?&;J?|- z^8l;_Ht)2D#uX>;HuzYi^@~`%I;VW>8;)fblFs%L1L2zK9|Vlrg38SiGx4>qgfa}- zHwL6YE#1SXK3^+U>Ns>%+|sg`r{NrLarnt2GKn=bqeW%2*+`kOPAZ$!gZP%eQeEZ_?v zELbq<%rh>L4n3)9ho;-GO5e1>F}suAMB`6gX;=iy0MI}(Y8fEZT}Osez1+A2)3iEs z`9rod=djvzn?j*>TaFG%r+{1dBH&RNX_Ek*_#h73D^R5QV_=-?ZT;7PYZYroOSm9_ zFB`p|KWRA!LIzb$n}khHt^h+{{IUy!LTOgwHBk4qF$%+|xX{@cX+t>q&iYt>HcnAE z&kN{y>9imRZu2NaM7x|VVVWJdwaCJCx?|nO-%4n44DM#%d#S<*tKv$lXJ4Hqc6adEA&ilwJgr) zSl;gb{x&Nsf}71GZ3?B)=QUVoHkmzZwk8+!UW5gG*sHm7QR25LdK;_=1YjRGTbz)C zk$(emif^BOP0rZa-kWFWnj3!pIJsGoTMXF(w@M`R6f07fq8CyFXzV}>STdFT29t#m zO#c=%j5NseL&3QFe3d8lR1VDXnuvDV7@mAL!bMo?_c*9d=l#rDT!0^3)$@uQy&*;vmn3nW}DJN%_jZaU1Z+~0BeQ$nGfBAgxe~&)hzWaY}6uWcZ zvn{}XUOc`RL1*mbJHHK_|1+3YG5v4zzfa}f#*Vg<@?aR&g;NQK|FmP)8!OPFhkOb* z44*xds}=s1N4gX-dIg)6cwHd?S@#-UPtxg4ph*WYfEUv1`6}P0{>bPVV8qVa$;T4= zv*0y%jdPKHSUm{Gvm!~c#ApI4Ct6=D%vj>>0KaDjI}8>N-hjX+g@#!Xja<$qyZlkP z?*$b%0&Pq3^HM*F9(&56KsrVFmn>37k|WU|TMhG6RJ%g~7FgKlAuNxPX&9|i+#qb#DABv% z0Oa2 zSb~T<1{iNPkvGiG`yi#t;7{$Q{upC_i&fyWdKBoT7&Pcpt8Y~}BneaS3x?d!mspcJ zrWPxPpQNmoWfp4kM#F>SmB_U?y|K1J;>M794i56a z<)0y7uT~3gHEt1T!YhyH_PbmzYVAV}h@*apRNShY0S2N~BQK9nopVa^v0aKsjkT!`!Y=NA*axeS^(=X$v2!6M53FvnP&9w0Oe3%f}Vs z39qE&+IM>8)3Z5RIle~YI$#_%)71SBYCt0>q-Da~Mp5>mh`8U$I<)YlvowEgNb^8i z^0RLU=HN)$GinQe@!CIh?K-Zbv!y&m!u`*WYb^Eq_{=AePh8LjaYKc8il9OQ(rqTQ zU%SCq`QFAd3er}=MtRXVu>Y>SgkTh^1b07oOGOmb(ax!e=W-$)^br8!_d1~eVCMtqxDwoqRUW*f< zV3*BL@bO&<&;G))<*ZXzje^YaH4?kIua;vMXg>(D26^3WpY*rY$)yNV^ILp&)CXyfCxVt&E!P64^!L~1WM{Q0?fod0;K+tl}Xre9_?9I5`B^>v(_PQ6lk)u!Y} zc7tPaLITEeirRq!TYR$wl%)rn&!fH+L~D?{qea7a^*f*vnQ2FjrxG%N3Ka<%JLJ#S znf8?O<)<0xbe1l&WRkBJfzhe^m((wx&qh*p^VI{cLzSRv5!LZtYZqBH7DYj^rjb~J zAURO-=Zc~F37>ey|1?)qmKpPI^S^FVLPsM5MPR1kDEz>A z@T@scJ2#MAvOr2t8VYpHC^pYX0dfv%GxXK`+i)*(g#8kB21SO~ z+RxnPxeyc8$%1GM(OqI#Y^`{y*Ukd<@#pV79i1DbfJnthnq=M4I33y~4X(lpF2O=Z znYtdbg!&WSoOss9ZFy?Fn_qiw?>*_AO1FA^+p81uC-Enc9%{wCqIt@l;r;!T1z^6VoVCLOzasrTdC1O#8>#&HO^YNd9>l>{)C-i zf9?Gx)%DI5i?1#zQV7m86R! zddcmoF#uGb);_3hX0m(c_pVmTI6x(?W=%$PIP4xx`CwRz|QuPrX z*Vh&K9XwVPm%VB%Fj&`KH7^g=3LpJuweTOzrxHzwD(5ZA-)$GWQqm_Pm?O(tIJH2S?MtQ!Yspp_Ts zl2l2l!FraG5Yphvg04e1&^@SEMTW-eoRsR-Y}WZR)h#KJOH~YK+1oada2miyvhXWT9C&jMPMGn-reRY8VL{>w8&exy3OG4 zo$!0|4Mt>v%4(tAyH-PBN-4+J#_H}WR;gX@h{=KL);Tw?iZ<>edWS;g)xW#Xif=fZ zlP&ogEuX2+xTW^xrAO&H5zo71=WxjW`eSiO`^;Lf+$ck;9pKhNB;={ZvXJ!0+Nz!Q zBx7ma0WdJdbgmtP+Fp)4ujhy}JWxGEy14sCbsf?dkWLQyo;oY~;J(^g9MU)Nh8 zR&ut9+M7ULclQBcHJ|kmZ#X1r=Nja+!1nIo1o7!|q3Tkgd(+c}7kVMmXD_(byeiHTyg6tR3q;{vSp@2_zE=f1GLA_E?P<%dT zz(a617Y~Hv((8j9554C<1uZEfD~~#I@(TPwnV8aYem=LL$A-eHRh`5_NUl1f4AHtL z0aFnaGDwi-cylS^B8z>ymdT29Q_B4E@EK7hdY!5~q8VG{Dcie*lt*x-^R%oKG^}zo zmE_1--3#@=s18?=XHq|uLDFpJD(#9GPk0uh_J0FE_Z4Pr6elcMYTTR2+F86wYH1J9 z72F{E1evA>a&aA2YPe}5we|7pm0N@xG0_fAwrazUpur0RFS^4z)!V^ z5KM#cj7T5M4?y!!0p75}dGYuK^8Cegfz3YmMGk&MzFC)6u(MG874m5FG6=6*9j+OI zo1u9d1JlpkD)Tz{bTs{~_H5fXYoTF>pXp~m#STeXg*q=X*Xd1xaMt+plwWyG9d0P# zzk_hhgK3?s==OI#1}fp=r4C#wdBMo+IWgsBTGG1cTZQIwH)_QVoTd#Md#c+U+F~8U zb9{Cgn#g;X^Bl_{Opt@Dd&x3|p~FSXznLGS{sqHfy9OwF*iutFsOPNy#~x${i;K&? z?~kJb4A&ZinpLr0-8U4IkJo1ckz(v+xmFEn+1_>BWxcSCN{^->%c+k(#F5}~DF%d+ znq-lm^cqU8^f*C0hhQLwOufMHljU*3x8O6a+nmPeXsGuvgf2TFTdK+W6iJy*(> z7M6jcBlM#qOS#I9`MYYg76V5yQfdFKs$`jZ&wQzHYWb>+;Sy>!gwqmW>|U}2@} zq(<~B&IA7glb*U~_r_aohg>2*<0)t&|E@qH|LmL0D^U7FK;1{?Lx3dVYA*k-_VkD+ zL2bB7&BETEl6J`gv6u<&xigkM)59C77#IFJeJDb~vm$_26ekwE3xV>zWUGq3V>J$DECL;5AxXgVCdCimgEw)WjbQR@zy9a44rLWtPZKDhZ ze47~eifZz!cq@R^MbD~hU+)gfl08M8P}hg|ue-eqT=}T~QBU@B6U`$$9_b)oyPzD_ zc;_>5AnpC;;zD>5giAHYpL}MI$mk)!ogruLWX`TWPpPnrBgMS1qJqms%%EZ%+fbu%{Y@ zRds{nt{2opj*@WtgL5@HS8EF1;49K{ z!r^SEj{_jh_I^FeRcn@(=XDw$!2??4Y1ywXQqCVIGX{xPwD|h)8h77yy8RwM!MS9W z2SGV7hw8^~mX)1^VL$ORq~%80Fi)Y&++SC^(6v*li|=6dulp=>F{Yhho%~ZIqvEo*w{h_cak&)9qdMusnlU(MnVs<^%&X%e}6kN z+xYnjTBVYcO3MZDn8Q{J7xC;$EO~5w9!4rh*gOEJ?hjl;pM`s9*eH& z)^QR-G^F|)Rzv(bY90~9CfWY`^f)~})cbE!@!U-ZXKNUN!#N2TXz`rBRP+VffVtla zO0k1#w`yfp8)I1?dTk?A%iuASHU|3+QB+iNA>+cQwf)Iu^jCcu`S^itTX+H)Q)<@v zrHCDOT95nj=QiH!@()np3eNm@SiRqI*83`E*P#Z;u>2DDmR6Yh@bzaW#U|}idn<#^}Dkvq)2_q`F zV?L_i8uy^_66lVA(%ftu$*X)RG2|FUjFZ`yC1wOBO ze9Y>%DqZ*xy8mE&873IrBx{jJFF1T1>+RIH*`0S%AwNC&NU)V&PH9`(6xKaS553j@ z#8fHM)*P;A2vfgG6zUs7l>vdUPUxRM zmGIjNkl{FAwcgw16wErHZceX3+=y2Oyg8yA> zoR+4PTCqM_aY?m!GjTc~|*if>ryO1f^l&AZo*luqJyaV4~b=`U=V-e2h8lkf>SNLM#eow1j1mHSlK0dy^VI{K&KwGe?d6u56@ULaqNaXI4 zFc)T_R{Mi*=A5VGLbE?^gfKIPxPcu*pfi)a3OJ)@>}5eujc4v{bUA5Q zc}Sfc>wXzQ|cYh^6Tp%>4ndytWWS%ig*a0rG{Kt?q={~;qZvog$87Y z#?;BMR54O$1RuPBGE|TcKk7ExlBjj>Y9Uxuh>5TXQsS5KouABStRMClWEl(ZjD=)a z9pbR2pX7VJU0s7~Y<06uq5%Q@q@7FybFhUGuvPX8tPjMFuRF`=NC5?&CGzkIfIx2# zQoU-`X~Oij1V@nFyl~KvbK_}N!BdQ?TgC$7Q*c`Rc+?%%Am9E zNAPqn19NtggFhr@WYOEm;Q>*Rewj-wp8BZfisKzqy^)P-2VF>e0p6n76>xejA_j;JL>SzfM$b zFXP$2eC@S^V((HMCHD1HCcXurvtZL_11XG|i}cnJHyJYuMiGPrr8nC8`p_yHO_MRL zEe<(^~2tEJR`wZ<{A_yu_FXC-4A z`>>i=oMuyz4wOSqcG=6G`6|fN5PEUaQBlif?e!aub zG)Nffl;|@BesFZ^u@}oK$@*fWAv}KQ&~4tuIRdnBd)8E5K>bdIV7LZvpOHI~!jD@4o7^iaC@v-6S|}$pq94UryL(e_^vL2=}u%jMlzt(|B1&w8)uUC$cFDg_rfWQnnSulAQ z2DTsbyR3^J@t591KeB1))J4ub>pTx>PRfMFE{8sDgpT%9> zUu&`VD>4}liVyocwX87aSJ>oxdN@_U4El(ku6fk%e#bmrU4Es;&th!tDt}0c^EUoO z4F1V*+VzuRmdsGfhXthQY0+noetqGZVf6F`JCO6*U<90jjKbEP8dx`Qb2cFe%FE^T zW2j^yl|f%$$NTm8(a)z|(l7Z8R3Ms=DP3kcajgCMyX&&yOBidXI&y?#U*hW4-(WDG zR@xOxQQ*+&6G2Z%sDiLZbmM*j)_dk6Lji(l8!c_+uYnNU7#jsujSw!>wW%!pN4l9Z7Kc^4;oKn; zLRa&MJtO%Lc$F`1(Kb)`{^`6r?`7&nf<&@c7oJ##2;OGwqB(|!2o@CGcTQAlvQaON zs3q*C<5%eMj5xIgA8==`_d#?uu&)_?B3jdO&>~&n@~N;R8_HPma0;8v-f99Fp(_)6 zxwoJad!;A>T%OaxmizMME-c>C!PLH9Vs48$1Co;M17;iZ%F7v6D8?yV>+2kyH`&kVT&e}mNbfrkGNHo43xc9}Y z$736(vMbIIuwnwo@QN1ch0b;E z(r>bEeJ2_QCrvi)95%w!61yre!Ev`(`3jp4OWpU-YpqSuqnF|D1Igm{#O4YzT0*i| zOChaH*1jwxMq3s?BrXrye4GnhT=@12JY!Nsb}Ue-_~RdIeXG&~q6Q%KY?sOgKpM0X z0+XdRk4=Q8h?Dw&pQna?IH&ud9iD9z7?=v9cLe@y^CY!9_Y=_`cc|9L9~qMR zA`lL*(Mzs(|Cot1YYwo(s`%QnB@%RLiCbjBExbU-eD`PJ*3o6u;hC=8Pa8^)M^v=a zWl;mud!cxj(i)$osp+q&EYmu1{Hv82-=o~*i|Y+5CnqN&EBx}%(zoKTHAoboGbjyeUi z*O-%QDNgTsgn`tIDJfL^h60Nzf^fBb++1>1SCQFt&{J4F3HCRBN>hBwm9r!&eo_Q7 za28ExL@20;INBYsTqX-LBe0z`=OX|iy_1a|XQ*GY;qdC%lzLB2dn26(&waMwT`?^J z;6|1Yet1_YrHQT~k-V*tlWv2#$`ij(q9G(*ZpvkJN59EIUyY?r3|nl6jT*--#fHk- zNlw2D1X2P&DRtq_8%P@JtjPk(do`YpaIa~6eF6a%QTnhAA$kIqEontJ-)_e%z#tpxyB&)LRc1dtyE2P1gTDa!)+%q&4G^?1MsW z-%RVc`X@8_OxsQXD$`2Ehtr^8R}XTmW?R7Qt?^ZJ)Oov3JoMv-yXZ7|tx75VK@D=z zZTivJDO(}3{ z+uQFf<(CuZ3C7*Pe)a%_A5%lUS`~PU}&ZniX9PS^qr=Xg12z)8v?_K}~R+3+i31bYoJnxPd-Yp!RLCVNvQ_?<36)T^596DN15s%n_!c zHB~Tlp!QL3{CIB(X?cR#LyFt6vN+tF?$CQJ0<^Jz3I4{yIPGBGAe+Pf)jVM)(3>$U zzXj^K(p=7Nw>&!zP6PNuHsX~lu~+z-|JG@2;?f$(6QT790};0EL9R}x_r8qk3g1&F=nlo`7Z{=FNd-*}HEyr3heqf6v_hK6;!9B=)YatVtSt zA8W{J`|oiNWKqrfgVSekWzwKg&4e#z`8#kOzMV6oy5`QHvKVsiRS$P32nb6PrJ(SLv=@TX&gud#XP3{pMQAAoecn)IJ{@n+ zwTP?O$|BK;&jf?5x!qn%seCwu4?f(p_!q7pQF;r&nrK5nrpn;*P;8YWViM3s z6lvTf{%iR*P&-IL%!C`!2RUv=hk^bwvl+k6i)G-ZVKw&8+>trZ*^S5y7JOM-d&p9;I~F?}v2q&N(RZ99(9vRfP=Om&46fG6fIZv>Y>X#FO@axOo|6{VXoK#R zv#J>0@A$nqw4`$h{sLs!d5-}p7m7N9TCxHqYW#<;(v%wTgyuiwQ^=l^NMWQzS`6B5 z|4ZJUOt`NaOGU|t@4j%UpFJ_$Z{yeqQpD-iE8SpOQh^x@L)Xv$XSk#38+Wop_WFXY zXny_Q@eZcon1Vn?IKwOAAE`VoN6>A$(nFO97EX2u)oaJQMJ~dgR2iu=gF>k48{A2c zeBh_5kT)BB-Y0>V`S`7_*SMFwa{W)>trBhj#5nYzPB7^_s^~Ld{|5lFKuy1-Y~(u% z*&T!I5blOVIjbnP<+u*AJ0A<#eEU%t=4kUZjXIjJw;0aqhToY|jUN;;-FeylRLVsy z`C0@fY@}m`uR2;V+rbR$q{Hjmgv~=Zxxm=@6t~)-xu?DXAY_>dRh6ufLHb%bR8}+X zpa_r+3Zuojb^=bxR3U}p3!ZUZn?NdM^jv8qs~ev7rYX}~vigY^1xx$KjnU_Z=#_^= zwxM}8U;WfIpHM*-E~o`t(;JSr;1{V8W)iAd`UNbK3*|0ZE^i=56w?6NUDx1q`VU#U zm~J=9q*7Yubv_8H=D>mHHg~^diOh2*(oTil`g?G@G|anouC53dN;8^v#OqEf$E0#h zDu@3}Do5`#sT^cdImY&HQaL7-W6vtbnb54f>2jgepsl}ux;u`xCgOiDn6yD7~>S&q>A z^HOS=$gX3zo48uIm(9)uP8 zq?kEb)EFNfWRC#sEtd)oRmP~&_Jfk8$4)7$gkt){Z=h1<%In{>Qus zeGjx+_q*4B)^p2@^PgE#JK8Xwr%RqQS%p%O8(=EM5-CzRqqmf6QVGqo`gFQZ)` z44%_-nW{N?O_R0r(VJcZK$`vhLQ~C_Rd&I&r|}aC(2O#xI=#APBq_PV`I-I2{;&D# ze_r7&?#6Gf>QvoQxw1{*xX(!T$>KXzJ>g9;{JWvpS#r5W;6q?-yGxpnW zleNrJMGBeDGIj%MKK@Rq)+Jv6075rlVIuQf3NOe>F|E0%GtqsZ4KqLPJ~@}^Rwu^# zc$ezyQdT{A2U{TX2;)IBxMD3lM~2Xe7Jn=zoxDNLKmf@?rgL&3b7nUpERRA^92o33 z)q-UZo76`UbM8T0y)}f5(k(3zi6aV@Uh}g>M`NbpKB@nsAkT1z%UcNW`ugluZG1D| zesuxUMxNtais=v|bA%XP zaC;u#e>hALz@ogJmAA9mY}OK8`V<~-#-TStYLd&;MWr+|Y()O1VMXM!PmmWIJHENm z-)oa@jWA|mTx_~OB;0^`QL6a>h8I`lmWx#0_5r|!;j=@?L-usMgbj8sGoJKVx7rd| z$7eO9M{VDgOWe5qs(9a2J5#RRGYhtqrTHJTQ4#l|FVf0_u${|v?xXO*I!lLY4hcA| zvj~?{-?m0s&`p&qJmI>2w6#l)L86BS*Yu`uX@w+gxnv0dp0Hxga#qr;-6>TiYeLlv zdw-u;6MRmKUWHPmXqZWP>8xyx080+uiw{!jw_Jged;sqsoMyLlqYnBEFK0_QARfSx z`=Y_k8`$$bRQpbXzvqWwhkET5WhG9dIlkd6hVf}>zO#jKbh*LOIL@C?-&sW|6V{KD z@vTvhN+Uq#%>~9SH=2{TbVF#Sq#HXI+xP3-%=Q3t<$^4vUhiDXW*K)cO3$#@N7Joo zM_-_#fHG=f!cwD0+8{lu!}x~xVS)a_KN7Elt?^F68Hm^%XS<*>%~k+2sM#|hUuVCL= z4P0Z9E7;L$-_DNUXODD|JrX}t> zc+bWv4A3;3xv41R)76sP%JO;+`GJ`KX9mSYIviP0P`y5}(J?p7nKQ3Z+&@%)0VB6h zYeMAn-UcV5PMqtrd&y*Ms~$IIVHHW zK^-zP+^^)drOOjoZ5ugL=XASY4~-bdb6Rf5*JrPWY|GA0c*K{DaCnRXBe+>+dd^umQ&WulyW&2Wej;_A>C`$jV`_MtICKM^w< zt*QKmoe*}z1mS&y*K4-H=x%A0fh4O^*ENonYkpBRa4@DVraWf2w!m;OncApC_XU!* zy@B{9cmX{ZaqSLH9W9K#7+zM8ZlK^2cyKDwM7207N81vUhQM8;E%1U~nxjoEnbBYGt4ytOMPHVtW17?KqF_m+==}BDi_`Xpz?W=MD+a$4 znp%J4knGGBYbmeEkpl}^alNh<^F-zzZpcjWm3m_L=*;xGkNpyms;5m*XW57w_;=K4P{#O+x@lfi=qW)*HD zCPj|s@68t^HFr=BOUal1@wb}Az*rWnaf}@rn#2 zI5D&8w_vtRS;4>r{P2$HTUlO*X2Z9I8Z5P(C(P-GY)#Xu$2|8<@``v7T%%J<+ZjWRI9r)2^?4Blxtb~pzrF*gS zK^L=Zztz!WYf<&~mltpfv7$Yr=n0SY^N#n1n}bwchO%m>S9c65m~b7$&gmfPzW1?n zCd{1vu1*Q8Qv&PmGbOP0K2riKnG#sX_HRmHof260m%!Qy8xzXVyl8zj0F>j&Z5@NW zH&9ND&^8t*X8;zbqm}yr<+NdOI*8s2D5njJ(*q7@(XE zSe)*@jseQqfyL?4pvD5_#KGcpb$$ZM>FDaDXH0s=?lb8bz0ag)kV(%N+rLTAnDmVO z=^1NUra|Nvzj}KMX9B)9Ym;(iljR-WeaE1r&~k2tEyFB$Le&+?t4wqB?k_e@P6xtQ z?)ubH)6AC8@1ncJpc|W<4SGcrAfpnRLES5PD_oE36zP(|Upk0_U&Zv;NTar3h!Oj! zNQ_9rb~8v;V>oSX^M5!HR7dOCp3F%B`S7WNX}F?)=Q%B8~78_F}f=sZ^d zfKxjW>%Pv(r6m8XSh*pqOfF1MPeAIuTvH(xQ9NY{EqgEy>`Uf*ZWg@+;aoFvTXN0n zVq%meOz4tkURdu(K`zYLx+j9GCUedOqFU#=(^#%PeWGbZ?0bQN?l z;ahcR6|A!T+b$;1Sno%QWr*I&O6ZGfaZdF*CSU$lt-ZY<3O^&2GF|G><0x5Hx+?qb zgOX*8DmIuN-S6WCapV<`$v`}OZQuP4veGY5OT@$FW^xO9Wgx`J3ozmxiBa-~mE@|< zgtwLm%!hxOQJ!D2yfEs^Rc}7xxav1{PdvdU)RX^kiSym_8rFDnmhx_Brmnfgw|#}- z7Sx?h!B9ZP9gD9G2xufh8kxVy@ypu{S*D1Ld)(*t}o%|2&Hb!u8(^ z%vF1%)ZJ^}@D$L9z^)+^nz;=#umDa_x6$K9&#~vb#x8;ECH@m5nQM?{YDi@Tg*<`6 zY=&wM%UW-eOAuA>cwXg}+eu;H8K7xTM%}qvm&pKbVpam|NrcS#OJ=8jgH2SYXF&J;z^pi`SAQ`>1^~R-Uo4^#V(Qr_<*Ti~;>9*V6pO?; zo$cyerWZ!_#pzR5*2VxahLBVeMkFCFK)qy!SGX}lP9nW* zEu4S%a(M4s)d+J4029k74l{=@XC{y+rZN@-TYuJrKIVAg3mL&!7;dA1;gLt>EQO*u_r$9z*?4sLeL zOw*-ir5TBt@&)V7ydoGEUz@2uSo!q?tw}aHL-1@MwKcA`tcwjEwwSkWjkk?W6FpYS zy4-}+)TaHLvl`%*1Xnqc1#e-Y8lGTQ7RC?eG{{P{iP~6JH9!_pIBjry+hP0<3~{io zhq&s=(e*^tU?R)TJc7nyQ<%_U?ika#5qy+pLVwrX?V*ypml$f!%f>*)m7Cs5;1+IA zbbunJv@+e`S=xKZur@1cSjDMMNC_^D1qSULqFp#8ZM|W0V-62HwtzArG%?)dtxUtO zZcD@7r;RZmky;gKmH0xCDj_-0@ZcRSFI>}``^%U`J(W(9{15EV44Rug3zyZ8aGK#W zs``G|f#x2=flw42yEe_2P41d6(c%G%Db7gelr83-FLorNqt3H&9OEg*rE|%qvBFj=}Vbqtb~cOiN`SuzxGoy zz@CqxGbfD?!YL1e-9}J!(iK9Aiq5pTc8d{2u}^dI{o0yFPQ?Z%@s+nxJk2hGCbwFo zxqiLvR`=E2E|%2>;vDd4=zL&H^c%MRv_n0DX5GHuOSno~q~~=GMBLb02&gVz!Aqpz z%Cm3B?WC<9HEu3{7QJ_R?EU@|e-RL{-SGjO)a|Doy>xzt7#G$giI5NO=5_&PQFMug z^{qv0oL;|$HvDWP)%MBxhKsazLe?-8B{S_{??aM>d%kKz=B#;{&I$@i>YIY86Tjz? zB%NV4t_!1X9|Dj%_gG<=FlTVKK8ZR|5LPUDaBf{6V{{>agXYU#zl?<6jmtRxWU2Jqg+HZfkbNQ z!8ZwbE?g(L@DLtaJ|PU)t+W)vnl4o!wbrqj`S620X}S8=w&~%j9j$ zY0$T_0mZ>W3UcZ4Xim0-?r5rpS(_#H`0R-}v9~PC%>O=&+p2ljxcN%Rl0o9QGvq}? zALDbBma-_UDLy~m9ybJ?ls7g($>t73E*SXk5WQ0nR+h5Ft8u%VaRRWXRShpa#yP}K zY4=jQU%>!p*0jl~kD1lKF-?!jw+bh=z87jwYmc@aLE6{Htrx>8+;ehtniXrWT~G6j zp$uJ=%dPon_4#Efm8y{p_S^kQZ=LkkNpEdG|H^u+C)wO{h_zJIHnMqxz2_12Lft`* zrK1=rL0h1w1ySI4w{$ZX2-n2L%2~lbxK_U|=^*Nn|5$wx+^;rT&z^ca2tB&T2m=U} z=BW4ep|ZYh^Lvu8LPr|biNAg3!pr+2sgTUveGAsp>PtUvU_p8crK`h`emOqT=kY)L*aNZ7~aetlO^VRZSg zD9`$%>ix%Ui^k~z8R4qsQy=A5tC@5wn;6R-l+1?u=`|HbGQZ)nr&w|Ou`#ffO|L;5 z^dNfT68su3SS_5dgR9-XxHitk%W#3?J`MJ!efVjpwLiRZZEMlW zqs+N>Ks&Tmgh988LF1W@mCl!SgewBl5fY`@T8gKv zgq>#nRbjb(oUjpxRWmr`2<)23oz;zS(}`shNpT?ce*1|Iq9Y>_aUw~QZRDIGgq(Pq z0VJIW#GLJC+y=uZfT*+m#3SsCAWC-7cmjw!+s{4j?>I!B?I#wYXVl=g(0qCjdq$BA zp!4o#P0yZQL4&l(;w}X~}O`*-H43rA-|j=eel!t!dre-D-&?g49L^T6PZKN(^f# z%HzVeG%2dGkSeavRP3A{FxJ>yBbQo-*`0Cc0G0s~^jw<2pPqkWnZ6%3*Q4wKYsg7)HPq#q@luB3q=Y+M6_qU?K}y3A=~W zV4+&~49}z)x67rj8@?IBlW0wAlZM5YX7nyo+RFQ$&uq!cGL{W2Hr?Ebq>|yGj$`YNpX=OnBfPfv3PBrv_q~M;g~O~16@ z-y!Z(ES%u!`I-Hq1$=4IqgYB*vSv9&9c7Rs)CN+7V~|U*e%qv=3sxGEF5)JU;)azP zf=gC{|1@|~mQm0(xea;`d-1}2V*n$`!q(EV3doa4y@vKhXd?_)!ZzAd+SoCmIh-i6A79aTmWx@Mcf zD&6kMX#tvSjq>E54q5{&S>?u|MudhWWd^JbC<7NOnJziIVT6_ou1gBQn7FQRtA$Lu znAWG!@eL50hN{DRWpxgZ71~gD2uodc#?jWH>Q)CT#md>L;wgLLZHZxdI?|e#MOdnT z!QJ&1Ezp+DUbBtzyt=iN5k{eKpewRmQFCnpm4s@xlBN0QkxFD?*AgU+<9bN~l;6^g ztuN|C6ooPIfWZp4b)!=&+X4W3he?mz3%GWB*#_ZMi1s?6anQQMD#8C*v8GCW9o`-q z@R_GVwUM^!JN|^wOs{3NT4Qo?Y;bZ;ZOR&1!<9=OwjY9_ zxh2ic>#UHX5^)<_{C7EbIjpu|2$7_lf%_()qp$om@$8}=BGVZQ=g*jT7AAIl*m z8O6YawG%W06oU|9T>xMKn#1s5QQ%*J7FveXNZ_wvr*{FZ0-#?3iaUT~djk9#3-$$2 zUJ=`1Z$Muo&>o``r5`6rdjAR1kC7ZbCNX+M%JT?2cwDOU$Ta6MDZV568ke{{GHLly z`WhGc6_G?4kw6(8zjEK$l@U-`j{%eQC=gj=;jy9#KMFL~Bfw%k1|-&aIII@fRe*Xl z6xJ@#RY>Ltz+iQOT!p4$2m-4Y;3{|$2H~%Iz^!5iAA-Kx3ONWV8DZF~9Z;)~I1zxn z>XDE_^40*{RS%d|h_CcOUBv-e1&Hs$T*ZM{g}94{b2J9PDzs!s@iD-w9s{`Q5kRX( z0#~6Y781iJU~^AA23Qs+8;R8y`zFAJ!{glCOs>* zuaR(1j|25I0_JHnQhPkaQ+#4i6ueU#xG92a0NSYov?+q<5Uf)tU{eIm9!RH01tI*G zwkt#eGCdL)(<1>fjRawezSM?RijvxXW#FZFDJT|j>Ap#a(cm)U zfn`R6%6!&pQjA~|1(M{!A4Sj% z10?lBA4QPw21n|LJ&GXt*|LV%(uh%DNcR9j3Y&w+gCNCU{Nuro#sVMJ(-5&q!k|a7 zS?tlU7Gq$HJb0oA$|MM3v;$2PMfG9uMPs3hM!*)uLl%vjMgnHKv2aDBF)qf!6vaXm zwP1&$2iyx!vDO>;lcM58Bm93)XR9MMS3E)s$w@-R5!w`D&J(RN$WGH7y$jROH14FJ@F_ldJc z{>tD#k%#Y|pg;Ej{0S>GqrrZ{sD2}Xej*$wj|1|ve}JFy;66C10kUI&eR^Sg0&R4I z`UD_*f`_sV=o10g6QH;S<`W0i6XABTHlS#jo&d@1ETL$Kp5TNcfX^6sp5W+tPux;3 zG*9UG18~ncSf1drYo2|+kUWnE$8+zXn^8AW7=q^>@H=5OYZUa31J((#v4_C!jGDBO z_jMf9P6TgxFwD;0vOM7c$LDz#B}>d5_F^mQTQDOU82hDW2ZH)*@yidj*zCBH-b6!n74oFYJGc)Qscmhku+CNd{qke8Lx(hDL&3umR+ z{alK5FJazlnd;*KyQ7BQJdyd6d^1a++1lgx@gP9=w4kL+Fj~l)+E6k>1duvFs1kJ# zg0?uwIcf1n+kK$qTyZQVHGoSP;ByT(tx=I}=BQC|=j5a4Jljz~B{r~JF6}<6(JNsW zysVGl2iVAG;=GirlBpnH1Nb(OzCo(NbKr9}rD+M^P>TO#ubMbTn%4j)vm&E{%{%(} z73G<^tf3bXMg+f)Mtfeu_ok`cjvHPYI$lJxGUtM3m}y|T_)ew_J7+bK7h7`_)(~jW#Q6;?j0+G5yT4ta-jz2g<=;4on*8T_ic+k&F zJ|6r6&EdR1N>e*W`;l9D=72 zcb!mZ?X!RlLNO6~CBohmkAsj|TWx?j0Iv;rW>sOtjYSrWwwB&iP;-`+RCls9ob&Hq zhU{N3SO=a4He@T{-+;jCMS?0N6Hdd?aIjWx@xkP69s60ard}3tZNdV(Q`q`97^&{; zS@U6O@}Z8aRY^sa(IN42=afH!UK$mIIxYZpRPgC%8+iJlpwocTsfaFVa;;is^`I{2 zs8~DqcQ38|&>Uh8L6>-tCzW$jIrph@HhC?DEY1Fd%=$bSTCiXwqvguw$9Trd`pU5h zdsY=;juVIWf-h6lhgG_W+Xc3M2vG>+rSqDYHZl*padH7h}80FoCJ!} zS7?Wg7B$To2@oAAYMLfvb?|6W(+QK|)ruB18(}g)bfl=ckF>}Ah?;J8Cn51SC?tA( zx0Ro3`C9C~U=LO$QdVXgh`m9HX)ha03fwzmJeSQ!&ZnGIC4?)zbibHhH-4`;>u??C z<&U)?;#oJrK@(+kP0_Ul@nCh|wgOBJ(PDC;sS$+^hvzlQr`ejV&LEejGtToH^tRbV zeIs*L8jXE0%QvB#iDS||GOF}>DHrUL$KY3N4Yl=k4ZC6f`xbCq7g3pK#75NF8Hx$} zUCJ`2`UNpJ%*-?x@y)1+Wiv9g5t}aEdve+&S~C<|#k-ytqi|YcMqV^xt`Q*7oQT27 zaDjdn^OrNF=xY3=+3(y;t*e}hS;=S$^%U%{QS-({3LyJpwRMurx_EOLGhL&GUNWkN z>HzhO96!T|ctpv^ig!%`iKt;y^M0vMNKTVA7xo@O^MIK`8xaX&*D6BjwG#m1G;|OU z>fN3^R6EldTaBA*tBoph&|=63wyK9UMf*2l6yg3G#N82lU-D3h(mlz-%VYP^@bz=K zDI$-L*@ric&B$Cm!=&etv`0)qY=+J(F6ILT)~KkI=(LAWe3F%vI)gGthW3R>Z3$K*JcW5!>P~dR4ju&BY*wp~LXTa%p z5&IyxpLb)DM=br@-GJsTB_l5A3E_F=(u>0INjD}O4>vc$M7F!;-Bc|bHmYD==rAK5 zrNloV?s;FC^|ANBc8U@#DEAlAoy_iair})|N}kneAp;Ra5hQ1I&0P1tR>pE3!`>{3 z2;y}?NIWP|9-)FzJEEz=;o%f%0GclK~>xi=juc>JWLa6?MITO@$9B=wXOsZi!*J@5KAy7bO7PlGV zX-;b4;l9Oz1!JkQsR&+ehj&u57vvqgwdaII#?BEmn>7hs%L)#@Ibf>F{SujLmRv_z zLF$)$gPgwaR=y4z(Qe+}U z!n0qa+(5*|8YwasYpTY`d+`4DJ?|`U_+y_?{!3(L48Ab%Bkni=6xyAP)&yKOZ`K(= z6mje7NXIH)uo3_wl9Cpk8w4A)Ou5cK1e{I7Qr?SfW0@5fUa@0Wi??g-pU=Pi!~7{6 zcfoGEY%b2i!iE0hmp#+dZD!XpbAGhI{Iu#f%Tyiu-x9tY0>|#PYqnQl&7i;;2;rAC zagfQhrpel7aN;k(0J~4MrdV+n%5`GJ1F(dZyvP{&OP#!b!j?;x=>J5GqeJ` ziOfz*M&8K_voux4PRKdreKmi9P5Vy1zGF$HdnMl>ix|KNY6CJim_Rtb7A8=$Ume;{ zKR=RoK6~cvnr)gqSBpf1wI_N>)|B%2!4tJ+`h6rEQ7HY-0>O0w4|ym;um4%mY;IUk4Qhvf>ZNNxZlQ3P&u)2^CbYy5 z2wPNKNgGyUxKA%dBl$Vj)9eIW$FZQL=1G;&(iL;v+=HFhU@1$;#ymgMKy0?u-tX7ZAE{o$Ww9zpY``M z&smC+`LW9h#{A#D4KjS0+X)SN&Y{auJLfxB-J+Q>G{SCe1$&ha^2`ndEIf%Gx|CO> zI>a(-xXFK*{YOd3R&d7f^xX4=MTnAJc2+G5`Tw)`@6B=JR>Cm8f9F%6l($M!R^75Z zx$Ns_H~A^HC(#qzqsP+B>{C1Yi_s*y8!I-z5TxXu&3^Z{a1H>1Y!U>!C6ApUwra;| zk%zb(ockq{tL;r*Fq&#jt}K88jF6jhnI0<#i6{UC0eA-1zH-!%$WaGzK#8Io+aWp% zL0Av%3qT2K=WJbOb@55QF`jMM8 zE&53RAi-`dB5Fq)hv1^}N=cT`OvA{a8eP;JgzW%jwL&fu*;aW;jPI<3i2_a;CN6~? z!nK;CCtz$Cyl9r8sj)bf8_IhUo-alo3GpuH-pl>td^SK1JQm_==s;#qpwQ zb=EaFBOpdp>1^(h3_h`^XBKo+$)2>MwGU$c>S? zjUJ=XB2*pmj%{bC(Bkv;v0~~#Q@3#o4~pK{tKZ*Jk0xR&W{ zLq+aC_ufi-?v2q>6{u5$_I_P-wH)bu9jSH@V~v1?__EHAs_mMZmbyU zw@=Eou3l=@z3xElS`63z$bR$LvS~`NR{>p=wE}PX+$xg`n%TZK{k-XR=UF#%^FM0~ zLSq7TbxGxlnrtp@{e70DO`15iQUa`6hEZxmTMgRhgXWCkjSPnPz4z1FckmRH%d-B!nqQ9%8 z6iqPqTxv#XS9)@BI&UmpFY$T3#9p{|>5Wyxtf|<-C;RDvB?Xsa`#`sY1i{KJD}L|V z2j&G!Sn8^4;yvaz5)41qPr-OOCs$DSnAX)wf%P~KTix;c0&x48akunKSf*E7U+j_R z)A;K7-ad_uyTOrSse9h@pwkX2WeG~}`cg|czF)I;EJ&LgArS}$5)jXyfO2tzQq}7` zBiAw|mfMp7b`O3jMSeJH(u*ICz*FTE8dFrf&ioGb-?rBxXN5jQtk#{PgtjMDZtPEM zA0YV}dXucc$yV&_NZ84Uh`Xi`7i?UWQh4-bPA*NcW$-WtD}oK2bMKj1n$S=4AFu$J zjgr`4R<;IX2+_5~fJ9J1YV-@FySOVWcy?x!+>GahEZCYtYkcr9`S-$Xc>ql{ukq`7 zfSzNnscIh6Gjl=U!6DZz%=7y;eRSrmhTH^FbSM=)$jm#K_VY7u7eX^N&;KTh$Je|( z%bhZgQUkN4aOD2kUA5Rfw>*aL9I zU=hTgO{=4^9Wy=Cas#atvmF!riJsC2Yc;vAPQD^D^6XS&xxi}#hvxt z6zCIC(C7fbw}gkLV&^Gv{Y-L4v2$nAq0}{$x6t+6SE+j(o-Aa_E|*I#c)7in8BgLO zwV;?>^F5D8ujK=IV7TpodcEtv29LTwiF5rw9Bu)ha;(lst(N5siR227`I{U=9BIK` zkQw>26tZ}MQm=sW+K!d=P9tcU@{78+7_)-7e6UWu*cZC9ZDg65@9;-|=;w*x%W#a= zXE(rOI2ur!L09^(ux$Z>T4TVxb)RM#yJwlZ2)3VvhvcD4tTF6LFH>WzO&Kh{yZ;fbP5W5i(sNWCr2eB`j_g@<Wca3q_Itu>XuhoFEVwfUh zbuO6me3^4^G5%1sxi(>)6EH|(fiVT4`Pn1?sfS{+TBY;Si& zdZqSLe^qFHZEDpP%a0dzSgsr@>{rm-i>{bKyJXLz)SM0jV#!&`jOtj>9w8rnZ5@|a42qR zX3c?}e=CFRXtqn*JD%D%7QUbf7ps?2Ect36{{8s%zhA$8fA#kDyW7h*FUYO`swP8A zzN!jS;TY5Wy%_T@zUI|wU>|cHX-W6Px-9b# zL>A=3s>qWM-2y>>r2Gt8clG>9L+sWD!nmEtjSbC;*^l~Z4B*ma)$M=>3qIQ$fOflP zNZC|#_cY3USpCTJJ@!Vww$BDIk>-+%o?Opnrl)xn1BHx79uWaDYi?ZfL|({yp7yFV zTo}t9`ByCi|Nh0l>RsqeR0SjN8zxre`USbzXMt-*Q`VP8GA3eiP*82>d798Gh($C? z9NZ+T?4*X<1^`1}y}y2`@6DfYZ?A3PiXP+|h_~ZHv7{C_w2ss{VqlF=bjKinBJBf5U82Vj&lF z!82ZN2{p|-0O{*8njpvZDmZVAw}_HuCaEc${*09CLRPCax#)iHGpi4~M^)I8Z#nI zt8y(1eK&x#!F29f!Ix0H7E1MpqSTWbR3KbCE^7w`wYM}Mk>K>wzGH)?p*$unsomeW zmOiGU#!r`fWa5kK5Z*^ke-Y!$=p4WE%HNnyyXELUtn!`}_nbYPKgdE$yfe)XXUN^u zIkdGn|7A>95It_cdG*Z;a+#(^VXG8dRvD^BYEEd*zrzA2XXK6#w=dVD;^`kAwU&|0 zQ5qS!+c%Jp3%w^`X{J(|L2X$OKG>>`Al|vyhPW0Z!_2)NkCpy%XqTcuv@Qkb!i^dMirQVp><3G<)5!taiV>=5ITRtCHNcr`>hExLH`kKuMLrXL>d-c zZ~tb_T%^R#p8^m~$@GEvqa%alP`x*1$FmJY$ zWW_{XZ`OlCniOPyQbP3D0r8%~vWq!a|O1v{0zmDOd$i~Ol?vmn*L)QV2#jloV)Eku7!iMx?Q_P zlN1B#G5xrEi*K+axh9AP?^Va<#+B6fX73d~p=6_f(H8)!SDVgBn$Pq^_tSb&`&7J# z&uQ5!D!LqmG@Oh*Xd`UV(#sw){q_bLUqg={D1d5&3AkPcBsqT3+mUsU*5RTGnbQ^R zkF06eS8Kggmt@7j`HeyI0%Bv}sZd=fTfRz@a*eazR+$ho6F*D0gch$1 z7lAeZ(G+J;e8V$ZWZO_2{td1KqS=|a=Y@pa3UW^i%nLAjN5T~Oe@?!;eE-*Xmv3L6 zYKqbeliAhQeNgB^)OCX!RJ^_Zu*?78fWI~^>x z|DaxHRFypWVhG(?hM^1L1$2zs?75=%^;nQE9=d4MbTP!Qx|Yrn23u!n?DVA1^PJZ1Oi zDy0`^Fa^Yv0S8WFom7#di=#QYvGUQGzi#(j5aLo6ZX%GR{p^UG$O2}RP(dyWmDyrmJ^KbK;ZB4gFCH^7@ZAW?H8| z(WV6{&yX2==JqrwcL(g)^v9}TS}utabllTz;YR;XUeS`Gi~|%Xltb(oq848k&^!lT z@r0BE~^kIr& zLisq;A1SETa_ZMD8MS8RCJ`<;d%7dzMoO1KsINcHKZRU*=$_wk5ezJ8-eUbUq?adk z`ZXxMH*2a1DcMw6$@4%35@JzaBN)_Awb-K-PoPrd6~m$_=?mL0V-N!YT|0rj#V%}y zy1J%p8p($to}ighxYvx62vai^D^KTEI>aUpunh-OqtPOdlX!M)pyWjV?Hp{Yr*oj0 zgL2QB6*~fxeX&<4I#7NY=Eauc1|RtL{nbIdIutas%d7R%oxB?UaKnmwp0ELh!Puyt zj$QAMOzD@3Og8*=6i2ULvHZ~W)3;GLen+|Bq7e&>!H--TS_H858HY+`MUn`hG~?E1 z->}O=f(Sp8z{QH3((nbYH@N>1JI8%5al-QTGOiA~e)Q{`rp5Fn%h%-V8z*6VUc`_c z0j90czIv=k#+NM6O*pFeqZB;UiF9W*$My|0!h!5i z-ApAZHNX28P~ME1c8&*d{{HpltGBP`oAj4!`7kTxtWs>o%MfRbXp|NiXBE?4EDyN% zNQb$pT$}+6I6x2j8bEw(Hb6cNQjXy^;egxrUf1t5cl)OC`;hjSo5=j zPE3Bv=N!%ZB=Ni>In7{7#eNAD`mV+!y?$CFp!8W(VP(C7rV^r*3OMmFm-5h=rW-Ck z8)`5tV_K1AZ~2jnw13xICQW#@DL|Uvj1$KBm_wy}t@I#;^+Gp&(4BXyz5Sc5`dc=` zug~(d-v7_XaojY@g$+dP$&7N=6F15q0;RcSX1~&k?Z>rblbB9fz|_4yO!K-_<$z5pDQy!A z*X;G#vuDq+5##TE_d6mBLP^RJzG*)g1dI@Vb8#^z|M&9k8wh{QONEV`@vHiQ2H-%u zJw5*S%{TArmR8L-L2Go|*$5mCPxH6G{`H)^;sq2|=cen7jmWei!$OvXLVG_Dlft^~ z8tc55(Ebn81o&!Qn#La*do!MtSPm9RryOXGLu}L^&cTU+u;4S2@jFJA>Q9BN^16>a z2FM0snx>*Si5eFbD?ecz(x82^ahKrfmPBZ}T--Iv^oE*=6BEk;p&yu6(qya&OT8K? zHeD+(K?PXkP?1a9S zd(OCy)Yh1<5SJ>k0~!10g$-y(uVJFpwtV*@!6BK!v}UwLpdwDp9ZDl8PqDHx#{^-R zdd^HK>}eZ>xuM0K))5UwomqjzaTw|+L);9QUA^J63-vLLFYg}L+k z_mB(_VX$wN+|4zLA#1mlo@gVNgqofD>FUeeH9Imv=`X;|N39C88DPsD6Fc4*uKdlO zTp33PZ!R%wEn_}2_8D#j*hgi_IG%tp;~2x9G4D8LUzKFXPygJL8Du(KzBbHI5+1^1X%U;S0tgBFSh7N&#mk#3k{0}) z6^fkvF_gM)ItulA{-ewj@TK@bWq90@X5n4AdfK>3T;h<5>MSgCjc<^k38Zii4OJ|Vj$GA;M%q-E&$FQOJ!-FoOxbm{$hcbHj5hjry^O(E zPGuizic6fgp~LpCm{1(#l2*kXeGEhU)4nhga{4l6>h$P8y{qT{-EfMhmKF6I--_IS zha@!1RJ$IACHr9+U4sM_c40A%Yb3Ng0-+lyRm`y+F&pnhQDh@A`BV@oEPyjYv&?>{ zt``uVZNbb12%+8NZ297yE&E4NTg?z@fGvG8RQjqOmJwNowSPF?m^{MEvMl(bD#Iu{ zrk9yNhrzRKY-M{!AlGQs9mv?sY;fXTvj_{dySTfAu@(wI9>5<1bz5Mynj%^Kyf2j6pNamRE}$(w<}d^tUc270 zpVAA~WneR9u-l%jAWC$4xkBbUq6WCw9!@p1lm($qPPNIY_GLKLuF$EbIcdff6xC_a z8(W4dY^59Pw|G@kIuvi)X)`-lhYBH)GpaM}5U$Qpi`V!Hl4VX47Ociq564&|%**i`MWDKq6;ETH zZdYKctBRJU3;u?}Kfl)d)>C1tYj8!hFjq1xX2Ob%B64}wh|4>=zZlTn?HJ*fA{Jpn z#qXM9w-lZ;m1|_F3s*;^!q=mAz{0i0oqdAk#u=yyh*!(#Q;aUOPY!7CLH2Yb4#*$A zj1hIcYxm)qv&1&thYisp<`xbMA8i3625OWD(iAfyz{a=_J?CJI^C-f7=t&k)&e<6E zp(kHNk%=e?q#J~haS_sHlD;?S7O8_`1WnsXDkk3#Qk|)uUB6+<4KIUt3|_o zcziscA7kK0tuR#Oa`rETiA1KcUTQtZLX{v^)ut%ldAhOl9p#q9`8{fj7+a_4>pg{{ zS`=~>kIaHyk;G`3$U;yg;~Nge15(#$ot)s0^CYigXoh)u!#1+mp4I)@le}ucHBX$z z(9436q$&y~%53|DGMC%Ap=9=YG!LbAmx1}z@pPcYCM_vu4~gf>#Ma;9p4`(y4X>Dt z7qe1+&y|cqmtY_I_K4|A-}g@R`%*J0N5q_z{sgE#M#LU0-rAhOlS|mK@hhJ$I%NR)&TFb*#I2lGpLaYz>9hNZmPj3}f5&-**_07Ys|@mm{H zfsciqxTdH%P3}-ja@&cGRlGdgqa))7-!M943q(^i1VYG^>HW5Va83>LFS<}8?&^sF za;oLTw;(1w`SFqHA^`kH_D0J%L1)wumyG9)(Hyar@MouKBUMTXdZsBSvm}{JXckc} zk2IwiK%Uq3u3AT+Mj`!0l>WdhnyIkCo921O6DWuenkpQ#Lz`Qu>HCpystu`>Msg|4 zGeM~4#<&SS*joNj9EA%2*wo*LWP%&FleH-bGD@v_sVY!5%nDHc4MLPlaPK>DQ@M)h zJC{-LEdDmy0@m&jDq#ZAKFRg@EM=WNC7faAW+UFBxZ9C?yvg3Y%Ftd=T<>xgTS9Fd zm!tmHQGDx+HHPga+r(6E;ZfD%aT(63E8Ov*A= zLIw+SW&1IDW+U%4a{eI5w+bDt{qC9N3qnsD*`W&6SHW0{`7s%D;EPJAGqO0fA%gvY z<&p@kAh_-gXhkZR-e9O>Q^*Zbg63*1q1RnPbDHon&{VFV$(?=`e0DIsGm@N|@Mizm znkhczx{%eXUw)(R?Ifeh?rtYsqa`bnZUr4q-Nm>%#ijB7z&r>g*RJ$3`kP*6lyAt1 zVvM}DD2;!)35&~UjVzk6V$!vb5?Bk&Bg#WQcT<30??C4yPaRry{<$9=3W&q9y#<&^ zr5D$6d3~jPZ65E8DNr3;d?v-pueVj-P|Xem;X>=-X+<++1UUI#kS}kp5G-G<%h>}1 z&o1)xn%3A!@^*k~|1DF%Xi8(}LuTy*$6QiP^=9`c3}W4^Aav>LIntu zU=-I!tGb)7;n-cHMg*@=xceDdqM{q@qSw)=4M&XpeqZCyE!*T7E#qT3j=uM-=AQRe z-)Sn+8O=1>U4Qp-C{n=P)J7=QttG3%bv@F8+Ne895CPa`!P2zuJ%gr{2f+$^E3UtL zc}Do0&F%jRg0x^P8z|)pW9(hZ2OvBw!Lv69Vd{Oda6kO3Ubueq_p^WXx*h!fgXTDe zejuVnhvqlCK3vQ6CTGdq^ym42iZNlLECwoUUCY$#q>OVe*nipR2{`EPv%t-v$wa#32d2m)!QX(j-8D!G2&1%qcInWW!ZS@7SE&Xo15I zUBHtR1ZwoNlvJYR;pij%ru#l@gD<39Es3fU%@p&3&8#>yg#29u)f5fNrHNwKUs_X) z?8?)@#lzW9VXkC5mK)YCMN*4GaIgafZQg+dlx z5=)knv;t{7rY${=Xv?gY?ndqCZJqd^)KRvu7hSUEu=7#_WTNB+vvmzLf}4%51R{2G z%ZClqnr5$9sp{ZNI5cQfF@cUN z==wTy<{zi32=)*K)5>R5VKqq`mjGrj8?jfI>ZU%&Xq(&bc~JZfJj=12a9{UMU5`6ui#~?7p~xcSZ*KfB(`J0 zG&!g%Cc#Gg-xwXDhogCi{{nqGjN{F%fnlx$UA##%@wW=8i6}tk`Xq+znBseGQg8e~% z?Tqk#3#~sW5mL-dvc7ZP7`fX<5Ti|jia5#62yHN1AN=L@l_An6LJA~%_-vfuhL*HW zBHue$8uP&ERz+FxJWfW0JaE*3{@owi@Zw4Z0L7Fu33QDo((ONHB&^%D0C-; z3surAt9x}D1dt$c8eftxh{lm{FZkyMLFV42$@7hT)Vv!E!OYJy+OxQNCDP3|6Okt!%~KueAt+s;sG^93w%d>nU0 z>tp7kEob_RKB@l}cTL+e>Zy}`6>~kn>P?6PT9tA`OP+uQkS{&&+QxOMu(1?zR}@4M z5SGkgnLF7ntw!Q~dm}yU!lUu_AnughK4B?DBafL0-(m;}c+v|Nc{w*)siV*acWG-v zIG3q6;hlqwlM;SKz6T$=2MNuqStcJ8DIcV>dZF|ecU((Djnr979WZdUc&U#v8R{eM zAOg5Eeb*mYmg)b7DE5HqJFt4b5}4i3l{HP8}C}~_jZLc1}K#F^&AQ;Fd z{e;>x_+!NcD`*LhPgB8@FA3e5YU3*Gy%9bdpp)4i=}1cL9lL=5=0uGF_VPv^7Zip% zCEuDB@Q%N@yzCzPK%5YSHUzW}py$Sz2 z*+9vTmPUbgKG5x4u+r3{vOah?L z2m$?f$T%KF#N5u#$qlnTbNod&gTSI;Yl@w+lJZQ|y!5u4e@6oy!3)Q1tH%$2_SP+z z2shp`Rp*y_!Owm`zI!0rVb$e!y;_p@ERpxDKm>UGF{eUtDZZvWi}eUuRH>ap(5kK( zpD@x3=?JTN$w9&H%l;VY ztROBGf4Uz(-}~|2mU(F15ai79hwJhRcW+3eNUr_f7@1sdcv&(V=Im|;#HXZ@_4WK= zXaYD;Fs_+>biFn`1pqYd2Oh}6Xb&xF3&J-!wnBirJu^9Y)^RX;2NH zxxohV9nJU{Eb>=~%I=joBkyFX|MMC$NJiqN9$@f09GO(p@^|8|946Go|HLvu}a{#y$daDo13gdWQaH7A!uak0vn z55t=Eyw630)ict1(%&k6Pcw%3j8wRuZ*sXKf5uY_Z1So_0+-u&bjP+wXKlI^4e{to z9M$zd8#Ktta1hgQ1b>SxUrbDqLJK@RD|EuoOGTe`^9(9;T!Bj3WM7MJjMo2JgXlZT3_DYgh zMJ|=;`_#TOag>w0ipDlQ6tkAy1%VV&leQcM@v>Q?%{Z8#@Rac_%kGVcf^I`eQEhX* zZg8rYSbltraW}@M9*e13l`8!huu?V`0_pI>ovrOq44ZI!{n8yT8COikA5f2LAuD0< z-d6cKd`I=_ZA}3;J#!oC#K5>oF7-Z{iUC$L@cfrcRup)4i)}aMiuz0{8J;wJ=?u@V zJBD*icASaOrR197wN)i8*%|gjTxPayN;DJN$doTl1mTV63_ZhQ5~d!6UxYadtli$n zbE4vZYl4?VpF}&Fu{(}(e;f@&Qa%wU9qO=TM@L^TsBUiypfyTuMih{-gZIiS-vK?9 zOY(JG8%Yl$-yqS%qybqHE|RQDF@6Y1sHRmhbbgO5-Afv&U$ruz&3AF3MX?0Fi||5X zg~Qn-uFk@!)kjL+F-rCfPABj{#J2_?S@ssOXf|X`n&!u^`J!Ot<(dk?GOrLby2g6v zOv{ocYvcu#Xu6fL@>r3zR3+KaWX**ukttj`?ScG>T=t48F=p$Lt!a-+utmgTy31sk_(S7GxI_aE^kOTbJHwy zEgwiK$peExBrT^xW`T<7nB%P;lJXL{$qTvRHcH;&#K8r4vntpUsd>U#CG_#uoo2p% zNlIF*SgGLmzQ{+SgIt{HP!n8Z86KvRc z`S8Hg5t{*zH#fnmzIqdg-UY%_CQ5y|i?Ez_0?pRv5FysZ& z_sSaj6)Ul57!-Yw<(eqIsj`v^CM%U~gXr`r4`GhM^?K>7s>1K8qzAVaz3VX&_>ccJlfoQ7!=+Of@kvoAl~OePeE-eIwA zRZ9t-yj>|yAjalqsb--IG*I#-5#hka=Ik2Tyw*O~xq$H1-(U@QDj*#n0h~c1GX7h2 zbS1~tgePlO8f+5~dFNwt>YAAG&SO3{pK7#~P2RKOp0kH@lkqis;N^N|l9kUvfIR;t z{AbYGfJ)V6nwp$Vte0x4Gt9jfuV;|jb4Dtj{$a=}HcI3fG+ylEE_~|t?@K7y8`n0- zJlnEpm>-Rh<5f-JwV-xdu|gsNl^*We`ryn87-+|<&Zh9 zDWBEvXzJnL5%VYTKe3e10g_>A&yv~ijhH7IP?oZA8)MrIh3T&iuZ3^xJuX->Q2jUx z`^}8L{r2kB*lp79SV0i}-Q21Oj~}>s_}?mD1%nd*ZgUK*!&W(Xi&hD%O{TB7x{GVW zwewo@YW?ZuYhr$l)LgplghPhHez3Bx7=K_-n^nSUS4zs&-65^L_BTDY^#$8`qHYZn z#J8=i$b%^`ZuFE}p1&Zkg{lfxHw9{aW(bEPwHERkYt5?{RzC>>9%;gCn<;~^9~&Ej zAB(b=wr!$ATE^4ZOLJP3pl-z@r+dYcLhlE0D0W^gZ9-4e56+c)D6C|=8giNo`55@M z>&rsUQ+9u@QhIR)Q;5mdZLX6la&&PtCpUbD#BnqI%gX?WsOZfw-mJ-|W`qtAzVKug!(^*Q_$8y2=F3~MHcQG3f&ynN5{-f+W~ zB!v&R?7zI!KS;%)l_@O=MPiav84dZ3(E2+kuU=ojfBo|E_Vue5x{o4s9-x9;30}G`*J0T&$)qshBO0o)RemsYq zjN|#S6~r-i=AxA4{3bOs=|nFpt=XUt%!wjL!59 zAFkGv>oFQlxwacpugppF;3r@BBMXJ?-5Bz3EErxa=nq(~y)IliD>fW+e;U$a zSV!uWB%f}B4;3Qa73&u>`@8TDl0i+8t^>&3JX`y8lL+Poo4M{YOP(>P-fekpzwcJm zrXFjVvvxduJKMHG`8-*y7c7|2xiU3tvus;80yNeUy28#dh4GZ+1;6JRTZIgPYc5t^ z6)nFum#J6RG~~b4|EsQZ(F+*v$GdNCp>Z$xiw{BWrSp`i?Iz>mu5Mzql%_}`Bl-^(e+x##`2}t6a1aI@O#RDIR%S3L>wETR zOD0UPhp)^Z9l`1bB=(t= zU!;$j!fGBzR!!D4zpTpjE3TjjW4wT}^QM=Bft`YlH-d1z(LqG5VZ8kFk0n0lr_wZwdVVaqZ_G5+pSDzph|3FRy zH~JsNcf|!SIW1)|%5Ttk%b8FrRmu(CFfe2C!MYXiz?gTn+yz+Yg@O(QdwwOfRx(t$ zxf%a_Mg)5>bK>@W@~_6M2;Dxi-@k}%UtYH=v>jiKttgQ~uwC}@v+5?7-FbRp{1-F46oI+)Le2vz5#-e)?6cAPMcz5zwp>t zo8E?mDH`$ommuAHm&jqJ2mU*JwMNl5kA?-IJ`R0*6h)u^t#7I`&Fp&r=5O-8hJw(= zYhE}%RK=tqKS+EEP5w3EXG+D?UTb6BPl{Ow8GL7E z9s82`ysH#Z9EIzA)szsjRg`M_7r0g?gUuE+>Z*-o<7nu{>!#I@-;Y;ss9k0X%C8(m zJORSE=ur1cJNdY0yM`2!-K`{4S*~ls*ZeHyr=ZLS?=^Wcy6wpJ_k&rY!ASf!XZEHHYgtOMM`lgei%qBfH9YZhxL{k25g5PhTuIPp*ziOttW&A!Qaa#{{+!b4 zAJe3N=vB?r5s7U2VY}y7uZ)CgGtrbmG7^DhWEaz(oTH~#ulBx}vM3*9F{VgI^OaTv zjrm(UiL_rE`MmDl4cxs}zrDUY7?y~g(mL`AhW#MU@7B74pV!sfpGyu@WpHdWxSGAH zG90XbAj*%RIrar($j}E@QydCLEAK8nH5IY5*_)CS^1q-|XdDr-Powc3n^a|)J{2ka zHdv;xLVvhDwrJYW9NWN?R6d9YTBLzy(NSp*h=@aFF`J1F^*az;2R>c>KNh?+wVzFe zXiwJ;yGlgul+_q*Q_Xb_gNgm2nF z2tM_ji;Fq=zn5=AA^pU(%T-vh3TFfd;saJ^D%k6!LTz}Q}snqk{nN7WzJXY^qXm2t}&-iNxS?e51Spc>P;zE4yTT9tAJW$10N z^_=`52-HVYlF)=A<3H9l4A+cYzkUlb z;2;ufQLJPFlN;k|f@=h<#o#$1uazvJqMwFr4oJDe;1EnQTVyf`mlME|$c^bKB2oe` zku{^K?LMu=qC98moRF9LGA?;S%cy3C`t6jZm6oebj_8^ZS)>fh<1b0#0W>@)@_GR$ z;ChA&UKX_2qVQtRP7^EaPY}7oFe~iV2-#O*ai5HCXa{(9Gors_i$3A2AmZ&_E;Xj9 z9W=~AhTTSfg8mVz2m_uQ36a~Y=i<~@IoBFoSbzhj*6lgMbqEO=8AtGFBxq<{P?cEqpbs#+XT{dQj)FVB!QsCBH^+3~j0Q;p?JGNa zaE^dGx>eC0JH8TIQQ+yxkpdVC+f=iNPRgEXpXmE zZ_X#{&iO?BIiI3K=OH~h4>+)W`gA^Br_QJ9)%g_NI-js#=aY2oe3G7>2XyV+vv21C zxV~TS&V9Ofj_u#MR|n6%dU)=$2YdAKJhYSNo~Qg|-8`SHpXU>I^n8k*p7C8h$AC2b zKOdp5=N|WB^b3b|_l%1h$I+?tS|r@dYDUm>WU5pft+Gco_0-r%$Qmmes(swCjlAyC zmKz_xhni~_6+0uT6vw48@8mMY;!jl`ui|?l{F<5W9>$3})#xjZW2rN}r#z#J%V!5vNItf~Rb3y8owsro8iWK|1NABV57uWMOK9BWI_;0|jQO#N7vtw!N@J?|krlGb z=rab@hJ3H}7z!t@M+48^gXVE>+(*!g$*bB`NEzLBTY`ZUoqr2h>66^4`mjD1Yn*4k zoBTxS*fruIs~0d~{4OrWN|_*|Sco8F3NHfK+ThGxvp>W7W97Kr4%Hb~ zdb)RPt9Dlr%TMfD>(p_5Jxr)aCA@Ep*-z-J1u6WBnc~XS+V#97^)*joDHg}ZyauAh z1b5hHu|*x~p;%evI9(_)pjXS3!iLObLyJ3>I>{>bl&ytSEGP>!C6;JASRtfa7V%PW9ppXrqCJhMQM%mi`5Zb15sO}z=k5c zT2Wl1;07YTVw!>55gP)wv#h9)u=a@8!*DdCL%q7AMME-wH%z`-ICk*oW}r6zEUEjSSgHl{2Ej!$Q5w6EH>C$7v~Lz0hNpe%@{(V+ZTaU09m z#WDYJX0K6*e~2~Q2G?I+UyZaTcyIRD6TBzBSc}3HS`@qhCi1FY)^}sXpJCGW zhqUEo(ih;2BUA`kVld1Zn&L+}5{U~V_Vcm}FZ*ni?6_~NRm~p`3R*ER>@KNf%{Fwj zoTbOt`ed$+cv(j>ladu1MdWgP@Q0tVc65Ic8EsjQy#pkcb}0vK?X_1x8f})KVrOHO zAL3O5a27O8?C$0Omo;|cy@l(P z-*YuSGlGikH>0iw!5W);5%%WWdxcmxqk@cxfOdCTm3dVfUN!nC4XLyJH{9(!bNbR~ zkXpaG7|v^Kj+~YyE5r-(|3CTRpZ`8P{lm%0e>|K0zyJBqCqK;LAHO>N!|C7c|M=(A z)030`_{-Zr-Cn=`Pk#FM{}9#Y4*%ERPyU0w{?AAtr>B4Te@0HUp%-vb&SWveBYgo8 zkfXra?LO>Y*Q~#8&oWLyAfO~@a{q}g?tbduU|0UW`TZo?-*|V+=i)*c0UXP|jI=T1 zOFiag^T(8yF)5Fqon`r_h*iT9?vYTZK(#M8Pz_JIVH$w=voUjxh*HD&vq+$^QEK4g zjz;|{N^N*dQEF3^+7zWW2Idr{HbtpTQEF3^TFjWD)TSu4M~G5GrL7}YEpl7 z_Ky^&rZj>j>@rDYCCXcQ$07nyTDF882+(4xZCi-3C4lu|7u?f^u+b{zT8-ZW$LycQ4SB7 zGKxb$Q69KrM6G3&rKDj0W&F{x1RvLb>&>PSH3VDimN0IjH0P%G%u~=Q_A%!LAo~qV zq)7KMTI}|$XZIwwxp0HEu%4LcZoh$*-z?^L_$Q7tt__Co<*AJc>BP; zix6*Gl38k7d;ZHat^X$Gwrd^(&BjVeviP_G&7GG1tQwzq{%f+9RiWmu#`%;=NiN_` zG$~pAx+EE+sw5ZB$cBrmj4e^%sVu+!^@wX2udM7@)WBCz;NMP5iLTsAKw;MSmP?LD z>1!*7b&W?A5nJZ&3rVw&d~jHl9|ZKbN-nl#T&oWAY_l}q|1m%Av=8i^x7SwCv=6Mi z2ll)aBde?mSp|y{G}Z!7TMgNI6riHD{?Rr>0`qq8qyWy~5j|Ohi>}@j_C#Wu{>wc& z_yr+8QrqByt+*L_P>?)yBw*-!sv|N)ll*`Xf#5m`KUD%5F}&6?9`7-{yt9=NdU#+JZe0DT}6Q ztjlamr=wVP=uTB?M8in^@DTk`X!hsmLFpRM{&ALc7mv zd-tW?!gse-Z)by1_K|M#jH&tEzksdH+^O?ok)FSQefjF`>-i=puVypS8NEL$<{+z#%+$v1R00v+LT zXLNwOi9eZ2WpNhnQMc`vYxyuM<*ZU{#)GkMQ479fTL=V7B=4h z@zu9#$Zq6`)#t2C&J&ie&kgkbiP0l(&qfMf$|AlU(Xo94xnLr*%=C(pOY)NCYX@?4 z*Xu`Ke?-Vhq=FpdM9QTMx+GdHo2AT}*9v~YTje26u^6bR_nDy4Be!I4HyjKTd(qwC z*XzIj$+l^@&w5g@av%ky-D=&u0{Xeb{N zAZfbc1NqUP1&JJ1$w02^{i|iF`)Xt=p&7gSCQ2(a&o&ja8}p;rpLswH>FW{x7l%AV z$h6;4F!cv-YD6B^W830Gnf8j7?19GfZ1cR`K3E-1(}F2wDX77c;1VydugISe)Zu-~ z3t6K0`ATjm4+bi6KNc82jh8{06 z)e@6um$NAb7`dYlDdu>A>9|M+qL@$|oPb^|i)rD*Rh=JzVpZ5O`$n#~Z3L##2RD}T zU_IJN4*KHdAf>X`f-bVKr?rXD?iKQNMpuNv|2R*%(*GqlH*dg`QVF9A1a|>^3x@u> zj8Y^Dhu>8sbfX%7d+!h~Qri&Js~=~c7z|dYF8v0BjTVgDu6c5I?Kw|mp??$JPtiu4 z5${FenOWBL7{h`0j^$WZFuO6k1b?GS$%GF!0tQ?FD_70NQeb0;uMUKr_=TYy)xm}B z>6<5Lm~?ntmkYZ$2QLQkYAD;k!&UUoOTe>>a%einPh@k=%wb!xb4^hTa>qR!Ne-OK zUKB{(eR6M;5gdE}?ihCe6Cv-!u<{d3i0v}Bv>faBn{h7zZOLU`WuYK6Z<`PHKj8Nd zaYK1IHt{;RpP18~-4#{&sw11rVl`+`T3a%ontKUbL6|XxuKE(Rf`D6vtsbLTRa9{* zlT;NOc=GJGzy9@{yy68k7~-yjcI~pD1p{Fevz$OD9VQD#h!0ey>d2-V_CvO+P#u~* zwP&_eY<5&!fZ?Ebp(6wRDwu(8bIS+zGb-ncyVm|xku*6I1b8U z9+$g3CJ7)e0U$a%c}y|QU{3OGo^b@kj!fji?BWQp&JZvvr#PC>#pMNZZgjr?j3?WxSL0&_@IA|Yl#3V8 zzmq>wH9!X6QbAW}WH`CG{^Q$IjVWL~x_Z_2K>N}be&F5VX^pKd7{z^e-h6^>c!bPN z%fE(NQ+068>vq8k3=Xihag-$?G2HWhlinS+kl~pT$wSCL}|2Oh{{`}eenTGk^TEFX8$o=#A#l;-0?sOsV z*?gDGg_OPrwiwuJMp+d!W;*?F}%)Mbr zRq%3)#BhX559B~>^{U~_ETEYotGJY5&s1ww%ZOZu)9LvJ2d+0YU<)Xw0_ zo&@@nzydS?bWz9$1!yQqCTZ$lPU|8s= zSR|j3ck)Ir>`{*PYt1GOIaC0!dz#V29Y^>y(IAvEfnhuPuE~P0hO*`kJKM&>p8il@ zF)`#gywOJm>wc7@|M#1lI`wC-v;X&-o0*qAR3ARgvi;BgkV9+E=jW7;TtW9Zr5`zz4uu z`;y7aND(NzH?Ci#t&MiU$(o8)I9P>}LS~s(1*$xyWo>jU3%**hB4k(?cFhbN(S~aF zB@iKcNL;UEzo8%brrHouZ5FK13s-zoWhE6%R$4W3fO{5jNq%I7H2bz;)E4gd1dvoR zC5}rI69i^WE)u!vN6Y%N-TO8ihqB>9uj>VQ7TD7M(3G66*ex%!9@|A@J8v;H!`FrM z>!j50ch7&|W|o;6trO|PynS=iR*M2s*|5@)up&@ptuYPLBC(!_ewQyHQOIRm4i)l^ zR)dy&^_B)JEACDY-|<@YyYkf5MLAqBw4}v~g*~lL8}N8rM>=BLxkn9jjH|5wd3!UT zq*i`ho%|6y9bX^6qk&~JSgA&BtaKl-oPBEH$JfDstT?jh(zN^5(T{1t*OCh%sv%55 zJ|7~{4E}N?0}7kXNo*f#hy;RiCIpF>1F+vMf;jQmMD??n?hErz4y zFYhVOKr!5XHBG8drI=s zO#s8-I8$@-H6*}->#(FegR&jA7-zgGi!Z_~%*H;tOx^hyO)7`$gx%g_x zVID-24HqL9fbQZ)T)^{#7J#e?rH7Z}ErKtlC<~djDm0=jf)voi^#W@@4lQRRRcWSE zv2!)2F1ykYRf5VA^qib?$YA!&B7vH#g zrq4EPxE#L5;VV1?t6a~Y9JB5GdqfO;#~*wCph{X+E!=7Hmbd!g3Wm*07qI=Qd95Sd zJIN|lYI*6s?abHS!w#;r;^n$p%oDls7}U8biCS*Y7nxj~Z>R>W*;QHaMODf|VU*Ol z;;R`gk~J?`QdR{!r#YV`Qrv4PS_(Daq`y=-OJ-E0Gn{#o+w_YW)JlEt8N|kR+_I){ zAxoNh5i{<6lFL+Kx;KhHuH)*43%yrc(;>Jkl>p5XHb;s#qWx2rXL762co$j2u`r=P z7d;i;qao}_Q%O~5fdi0&b>Kv3ioNn{n`q&5{G>X}{_#FpFsz5EhRUb945xaMXXG+t zMY)0GpaRRxQX$p*Dr1F%+(i$DWYml*j8=yQ-15j=!rj+wauBoBTHMi~XMtv?gB)walnV5$YApKC&5UaEsx2!F);myU)u8d;A@H%nO)A}9 z_N{nuC6@aAi3wBmJ8Dti?7Y^-;`xS^G^Hh-uiP{J$7#F1xh0YZYWhP`t?yR36sZ1l z@N8)Y$(Q6q*H=FDE0)S)OtvK#&(FxB!q0w~BL>ZD8u1_hIp5uzAn5=1thEHLh^)X< zX@Vr7xvQqyWeb6O!8T}tUHTAuhx%3z7pW>3RgqZm9v=0(3p_f0cJcC_+1xbvx0}q~ zpDqK2uO`3n+91}azhL@q7Ytq0d(I-9P`FD^J0HV6?EI7dd>3=xFD0?1eq~&hpo879 zZO0Mze0>)LesDwEU7mBQ<=Fvd{=OzlRy&|#ODIP9+H9Xnq^!uGc|NksE+%Wt24E$_ z)6H3-FC9zg#1zBHSjil4S3cA|*0Y)2GIz0`d?uurm8{rs0jbX*q_w53j@5MvH{_&T zGhUF4E?B0>hL(w`$p&hbEKVO&LbE_44 zxJ=em;~X}ih6djVW=>wyWNm-NUL9R?rbJbn4J~R@pMKli@DY;ekboDVBXq5Dw5HI@ z_}Uu8uD*c#Hp~ipD>ufl!UGoHiH4tI*6@k`S`3XzU)farqJdC*^UO*-4(51pqVKb} z`lsDs)J@WZ7C1np1Cp{uA{3g}^xhW2Q5^a@z#~-lr zj{i%wa5gv#7@ASmUjgCx#t5WPC5Kphf)!KUvwA*uMfOpBc6cT0$Kdt$g9fJmsL_wf zfB(-f{%IOdF_JYcN_CzTDg3`#o%_htd`makBYfl8vuDqK_nY6~e}9YrefIp<@ZaZO zef8C^fBWp8E`I&n-#x#$cy{sZxBv9);&;FM%~$_Ko;||12S=q!TKv{r^bmHmU)v!=WqAV=D&MBe`b6Ac0Dso ziuBxc*5zU~Ph`QQ(qH-O4(9!0HkHBB%i?8OGer(q= zuV>tN7I)znCMJY5*44T6UuYVsr?V<#mA@d}4^~Z#aiUCa!HHn zw?EtZGk*6KzT5b&FP#KGjHZRaXOVaI$s9O?_g?F}uJ7-{^ZW&WIQjq0q5R{&{Tn%c z&k`vTp0V`!^grhf1FZd&eV~7z<3HyD`Fg(Q{vP!)=Tm{Aq{gzWO1jCLKvzw0&SBR& zsPDa?3@b`$^#s<>sdbg0@JM9gH8t7Yfrf~Q^SJhZ$_p-`bQZa2OwPBk;%oQ#fJ)YE zk-f)=Ww^h%_H6o`L-FZpl4(*b?VhWFqc;={V-cJ z;c!*E;<~(UatR%DKnR`_jJZ*^g&;&26rerImEb=**sfV|MoYG1w*$H)Y@?bp*XgNX z`YWUo9y}}n)EUXQ;-zI;b)dwa!4XbqKkiUuy<>20!4fVU+vbj)9ox2Ty|I%WW5>2_ z+qP{x*|BYXIp^FT->v(jSNHTY)9q2@EE_W=g67G-%TZ z58?g&cGN6(hX9WFy!9t}Sv;2=4yHZ5SUhMGNQQh~$5GLPNX7LuB+D6!|*V^AAMjW!8ZIY_nza)rk>%WkFL7-Ocet__YHFXrPaQ%__)!L$C~%cuHPM*W2;pFs|}E?(!kH)XJZ` zQ)3dIB1I?VDOR6rUu-w<4@Z`nG=@V;S84ZjLO;3d?6Rc`*=t;y;j^~IGD~w+DorW# z_)+2^lv7={VOA2ywgYH8x(;-v?*My=rNk7X#I~O7znFWPs4Uv{S|s9c=xf0?OcxN7M!6Gc;N@WU$>co_sfz?3YMyc(Jl z_H$d?Xs9?dxt8KfmEnE11(?Qy=4c@EvpsQQ9!k)Z2%pS)z_x8W)7SeCFyVU72qIDn zD8nw&S2U#X&%yL~ZBdp%M&9u@LrTkFBs;!i9dFJ(_M-`lbB-_HJ?vRtL z^2+3+Oa(M1>tP$F65QU_N)U|av^Gl1dd7;z6H*3Z+w&|Bnbfi`rRff48h5C+7xi3( z`)I73a^^)?ds3=0$OxDYu(NFfLcYa=Std?Ey0@O>IG^M<^SjM3@#gIj!OR`8?fbW3 z+Xr;>XS^40jSFzUmyp(@J1n!1-;8Rl-OGSCdKl(H4T$2nLgbc?k?X@(EU;dk7<{xk#xS^zWFNv+0lR_QpFRo!BOLCc`K!!MxDl#!+o;$AjCq%*2Xp3p%yqoH1}HaN~_a-NZH zyxprfOa(iVvQ@+BAE^Uto1%=#Mc$A$Wm|FYva z_pNI%Yvr-y?r610^T1oPQ#)1~z$Jo7;=z&x!oGciqU-uv@5~`93wQcMFb8aSwwf}LZ~d~*!&CRzQ!T2y9QJ;(BrAM> zeH2et$(vlRejo-LiqwfD0Fg5KGRSuqmW|HE=9^dk52x^G$QEDP8N%;k+|W<+a zM%Sd2Qs}B8Q!+F}kZ!jbvRWw#wlfv-A3{~FU?6v8p{6Q`QSrnM28WZk1-jqf;PAFd zY9nlKe-I;UmW#MZyrz#zE9MJpF7vnKGRK4KPzu?^Pk18-I;=%>&%+RXS^4gn#lhEG z7u^l=>fI}4j!MjCG3uzXi>)n)B2$`yt=s5~oBF*S9=VK5zm>gjRzVoS$!?(7Z%5Ia ztJuVlhqwGQTMaBo(8yrX9F=5zXcHdKJ&L|3{J0fMt~fvmSG<~G#~xgPMo?Z=0X z&qehPHWr%nz!N|Vfxr93EB*_2cHn8og9z2Ypq zm_1-!D#6QeZ8B_i{UwBsCC}>TuAPg|Xqy=HUn$+9R@-tau+zE=v_rcnYcD<5vtm}o zt0gK=Q8W(4s@9)5D+F!x{)04Qi`<(1-lGcs)10+MMmzYsx%Vg(;?>}&Ehhue+dJ7vSl$um3#OZ`9TTJ9k2-^(Uyp7s?_9|mR-7#t#LyYQtTO1f|pJeLQWTAy0ykP zh|p`G9S_3H<)UoK6P^#f{f%Rf-}uLd&rmE%Og%s)_EWC_vYL*y+A>*>eFg0RuM`;& z?VMhpcu<*2L@iZ+KzykYw-q$IJ!0C*Qo=QEap<{%1J<2&%``f*Wx?m%EnR>+#0P;q z>;J7*XC!^VOJjLCowNwQwdnzbxDOLiauNaJ&q=BnMm;etqn?5sY8=vKSz96`q+}f# z$+etY8xs-L=-LPmHUXvGfO_H5xVvGIb9UnwJ=ZcPj`R8=@!E~r)Q-U@7f|TXT)~)5 zr_mOCsZJsgUFQ_nRjF%SITX&H{_r%r;pm(V> zB4O`u$$Qiu76Y+aopY%IklZT81an?0QKGk0=jL3*{> zccu|cDVDy4!atDf(OX#%8h~6Eh~zFPbd^D(lPZ%}xlJ@wZEjU&0Pnncur|Av8SkKO zS^c-3FW^_dnsJ$2#&U<#m~DuPxv-(*gW@Ka$pGGd9KzmR|LLb7%I7QddlVXo|5L$E zJu?Pq(yJasXn@6)9fNeKZo>h|Q*JQX#1XelP#mVygIm6o^0llXbP{?OgHrA%Taauz z`-US#&6D^^B{KuAYOeIW*jloJ)>im7k~ONaKZ>yecgu__0zA3*H<=vga%)}eVY_z| zi-pzhV5W|#)bQLUM&%&u&7A9_;gkrf(vf=pP^B0yp zL(%>?9Eo8@{4hh?Z3ZF8m-zOmTUWnwn71&Rzs~?%a0(lu;kF)X+pDudI9 z^uFK=gyKkW^&KiLDaG&GV|PpKXA-`@a+Ht5^EHoTtsTW=r%T7MS$W9K&?0{{X92m{ z;z49R;Qs(icnQpBx4-x&u~jQL3zCfW_a8hO%HxxH!d8yFf^fdkms9sy4vN)ya~Z|` z9-8*VF6tIic)Pge*ljncj$(50xY0c!Jv8jTj3$MP$}aCxI6e!}QEZf$7PZtg%gInv zhT`H;tz_WAsaFP^NUed8$Z|L-om1`U7C1<)RG_WEzT0V0Qm_;O;&^-B8V&h%dezoB z_(JGZ)WDxJg$(YfaRRJl;xsjF>ZR9x>w>2hGcQP{(=>cz?d~4dtyw{K9w3j-<%{}XwALrYY=su>^_yH z<42dJXJJoSx-W?z+ZVriX z%0-(!33~2`$rLsRoAf<5FONjMKR8yzHft;}^P!j-z4t&lo<`+x$(}4jy_^*kmu)2D zo=-!)#9Y;P!}VLrDfC}x9etR?7D0mfWf7!dnzG#zkz5TM}sZz`bfU;xHJz0%$5Xok=Wipl4pGRGlqpN`rK=?Wkk zB@QD*vTD)EVK0#Qax$4sF?QxY9+^#_6?kQQE2HuNo13q@#u32KQSThgoiL^ zX|Oy+gw2{T(HlKx!oWbD(sz@p1CcOc;D{|l7ZAck=~nXW_8e?CL@9F7ZIsV%;_J#AyNa)@<{;*a0p})*eIv7h{<5M>DYQ zq^0k|vGkpCHC+ZRDD8iiyByt{sZ5gt_{RniC^^2dx;~!Wr zTl-<;Sr{simg@xe5-LS&L$mhuYBmRO?$kr7BCi98f^JJ@8O8J>s@Yi>GedDUhq7PC zwx}%lRWXR`e@sU7)`(v}M(dA(>*hsm-@^PqAz-gx zSBQfsJw>Zw+YArWRmZnq$mp>^t_#AQiqgDe+A&Gpkx`|(pd6y6g0~(rlp_h%gnL07 zgu=Qzzfwk*n);J*fa`rH5b0T_nS}m);nS!vNESBHAGv#$;U@HSb_*#f=4juTs)b`i zrbF8ZMWOIv9@36bClwY117BftKXc%CP6rqD${sMS*AcSY3cBZ@5beLHicnzSvYSox( zd|?z`FP#Qfoc4kK4qLC;u+#It_tz;dv#3&|YOZ8jXfnTwYh!jC5ydHyQRDd>xCLCd zP0qD5ceI(ptNr|Y?oZvY&ghB=r(?bc}n_=|7kmj({{rW>x%}9{a(tlk!C~ky*Zn2 zin2b_+-TxmQ|it%o@F?GRhNet7z0RsJ=*7kodZb1K7~}kB8dZjE=??XR9FXL`qTff z$6b*rBtLH%sV?b6{w^{3IW`}>fgVoIH!M6N$GE)w8wXhU$WduQnT8fm&*yws{@Axa zoizP+zW%Sr5qaA1eOt!0zx6lWM1q|=>?^-(6I*n^u4NOjdR%L|>{d;ivFro=H6;sj z@3eAlGTB6s#nm?0#MCype?(``hpOT?88RcgVZ?CT%9GSS-fZ0PV`SWL6bl`T7!Lab zsNWF(TOF*YDtw@nDNJ?%feol7d?V*OFO{*x(=+0!jI>~@4Gt0HoK#{ySuY_5w(8{ zw-Ab`;5OV?oJJ41Z$Ld8L|B4b%znrW^+ZeZxyga6(L~`t(0M2w-BRc3=ivI5OT(6H*0_&q|Rk7%fNd-9SQn#IqjEi1xS%<^fa(1*c~d!O02*N6Ka*&Ml$KOuh` z#+sABR|kO7@oB90Nr{vYq9{3>?b=4S7p`l7;+g^@=`akuD!iTzCFKJMt!2T@kbX4{ z`-AYi`MiBD(U+K+9jmMVB<*|+&|}i?oXaQsWiS~-&X(E+CUceJZU9u+KW@2|*?rk) zLAb!yliYUk#cx^iiv^x(8B*wRHnnH0Q6}{whM5JEfe}Ba{kQvQ_`ul(e0^t_CvE=; zjZbMi8&yiDJ!H*{!6;*hHt(xXGCNX3(5-de$7Sacy$ieFCV5(AVB*?G<3pyXG@Ju* zcXRQ3y8gX%IQ-&(HO6xxnRpr=trt(m$03>c^#)yKa$CQGHA@P4a{hI|43wz9W?7!tzIm-X>^EFZ%ruCKzsvt8=W2U*hYLbDu! zJ}*!>Hrdt)U>APvVqZe$uqNg~#CG(EI=lh0)nmg0fR+kug(D2VK8$5K%RQ_Ra0K&B zBMhHe@$6_me%R8iA9zbO;85ABqS}7&MP)l^8_4DZ;@bTj<2__X9{YFhc4YVAjoRH= z-fNyVq3Qm48$}5iF<~iBEJ?MCd86uC06&L#m0+q!+4ak+H^3$%XJaGEFLUWO^i3 za#s2xrZTSIi#rJ((>x>gH=4NK3+L2ijQ{ZD5fKWR2h`(ethpT&rA0LI5?C=P@{K9P z$YZtf)Mk9i!6=~V)3_|gW)Tt#VAa3FtKOG3(dc!H16(#sj#l0Qqw#BUS*N23(9*t& zWL;O9ld$2=kFif(iMQdXjsgEJNuZmbkt|4FjWi-ZQ^8mB@~r|jaN6t~@=xrE(7+}c z*aky$9Q@5R4lXd+3I-?Np;EVHuUa5j*xKUyqPJ}3*e|gTi>O%vf(LR9r0rMCzQS$m z&)cNit~)JInWyw1E<1PnpT8j?Fu&@9x;z9Be!rq`rUAWy!FUa&JgrkGZ&9n(PH1o& zT*Yh!Te)wQgr}IPpDh$#>GoJ1)G12ixL4wn#MpQpBD-0)>MeUyn+z?)y)TX8oJuUl zi*Xr^NQmR;uW{o;kJ&dZqLK`KzIxK8QY|X6H?s`At^u!Z6H!>mp;*fb>JqcB`b1_% zM)(^LA)8Q+L1CbZUa90xFXV9VB2;pyswP=tK7nxHsg7GoAkzaLK(+Ij`CO1o#4G+a znRu*gqt}EiUc+P#gij!-+W_u+%Bm5}aXDncfOG*6Q95G`i4Pyx*hnQVl+<#WA^o9?yC`3Pu*sIN=fgI?l!*%HVt(w?Ps1D zrg$_8e|BFzq$=#}33d}8#;wZx&hQR+iq+rm$husiPTz-=a3b#tYdXdx60j!*ixTSk zux)nT?FJvUMS+-@;}-~Er*9EN*T;YyQYu7)97?!|`Q7c8a593JFzlZ>I96T7Eui|j z2{TnZr75EaaQ5+(i;v}cWO_%BrnKu9s|ZjcO#c+j5dVEjvTs(v%+(sdQ^w3iM`7{3 zCV?2?bRoSr_h;?ZS^p?HRhg`IwX>aRilSQGKV@QRDMR6H`kAKx=fIb-o#_?Hj-wzv z0>H9g_KB@TYwABE`MhTc%*|e6QBLIOx{s_iBt5%K;sOmbsaH!eNaErF>@~PPD#y+ zfVuGd+|l8sxo%QO9OFaKlWqpyutVkzXiQ|+AIJr{toS6H(q5=%1()ekEU}sCFuVXX z3K#MuIYl!ZMsPVjO(?NmiK8v;EQ7QVet9-Oj#z_#bcMwRP`?3?`IOL}EzLF(uXEtGCIm?LnnARhm3rs{rj0stMMT&nZxZQvL`vk8 zXIFpVKVY4*h!9vz?M?Y=NdAzf+_R1gU?vTvKlW&ly>iQYqH3+L;W4Cfk(B4XZS`2KDr^ zuYxbd@5`KP*>JxW%=Fxi)0W;K@GPXH%tf%bLVqgx=Dm)p$5VIM)%5=Bk2vc17QIJ3 zw#_UNZExz-Jr6bB(|LNTg%WsVRSPMbA6DgB-eHR4vMANLKv=8pc2b#d5v6MrZ=~7& zELA_d@C?KjL!}j44F{5jS;AkL{)^%E z22wjzZMIIZ*FuHZaEttV+U`Veoi9BK71UL7-r_*R{zChXHjLDmN|hjYIX5(8@^+;;AUO zvBL10j|`hoz+2;IYd>T_M3HR>D(#KGX;h1pAF`}Cm7&*Ayd7V*PCNAJ4fPZ3wV&RO4(q`OB8aDhiU$ELTg*Re@-n$!jW7OP2#va0^<_ zkHA$)m7I{#KU(s?f~gToXptw=a_PJd*69i7aqn3kMvsDF+Fouq6SWz_>~sk@Nfn=r zQEcaAKg={E>Bx`ih%~fEQWM(;XTK;EpXm5x6mIu9g9LF;jHW|%;a;sq#}P`?&dZ!5 z*l$_E3f|<7;wQmy0PkC#1-&JUmP)lgPtiW)^Dfg}JKBM-FmIx@7~BlMd87SfhWnX=5y3tHw7~DrK^$J zNF>YBv_!oPWVgfuUG99R>1%dA89Y55-0=$`vT{LHh=XyQ07?`|P|5*vxw`*t?u)V~ zh2}$SHC*PR+%D##KF}p$iNUwC4-}vTlnPP|^8ZRAw8j@D($oH|uTsxsDgLBZDHp$X z06e>0wSY!(W;Z;#Y7h!q=O!!?dszM9eiw*A$1dH{bR%-OO7NS8*jbrn`!1t-#i>mJ zI6X}aP25kMN2p^SUqAh01D^PY)H81Yfz@tB8TTMt z(p){dIc1{}A!uC-IgnG2^w4!Z6eU1`RxLL)=+&OiDZ(^vpaIX&<+{y97I#-#0S>0Xf`%Qa=@Y5LLEUb8rQEL3vGh$kKTR zdjl~m@tiHadTyxWVJej5)OE@uG#_XDOE4QKTet6-dbQ>Z{PmPWJ#{4CP0^ncD$)6azz=`{ar3_dIu0HYPF7S17HcEW~z4#yzw8= z$&T9|=srX{C`j#fVt&}ibVR1JvHG2JAiD-Z`ks{lK72G+5F1|%j0jtM!$dm5bl#eR*gEd+{L`67 z9*xfK38q5OAHEsL_e;*$2==+CZbFRe(7%LtJHsPTYIS9QMdki^3G)CrOHgcIaPTDn z7x8Lg&Z-fQ=Yg3~6N=Y{JHH!c5A=eWw&e{ttF(rL+D*|^!`V%`op*X;T1cG1^6wqz80p09xuc;U#vnG2VQlelmif3CT$=Nm}(q-Kad*Anv$c@ z-7J*37@YBEDYH`N1#`MuuezG<7&50eWQ;QzJ+D^ug_gEgaxqi*6~;MShCaUsXD}} z$8#K=PHW?A8%^ppxq8t>|9*w5H864|;^pED+Bf=3I~So>@u6%CHm7-l3vO#B$Dz&f zP?V`v*2DM1ih}hkihlGWXFwS#?DpS`pqpBVfoU0}Yd%)xVNnx1j@x?{8}df(KsQQM zg^)vI#?lvY3>vTUFR?>th=QpKih*0oO}O~crsbzqA@q>z#d607YB(tGb&`awK!Tz4$>ItR?eSeSG&m9B7yOAgJL2u7 znEKQrynVpA9(tW8=~kOG%P!^|lCwOJaT!}@3b4_YRTlbgMxeIsBu zjA@TVSq)esL7KgSO<=y$_sNWuQz8ag_49M!ERQOFWo6;04v%Ck4y9GfBWOoRs)G`K zFVg?7+s)oC7IG3^p7t~Xn)NZ7Zbtc^r4hDHk;p+UiN`q%Y}wLrHh~c&`XtG*f$^ig z+2XVU7IA>anr#U74y#=1d2+S43nU7dvUwooELUunWO2EOU6o zu#^pXvC#hXb#K<%fzOqdm4=$obz&Nv_y{JTrsPm?%vZisY1e>|?ls3U8KL^UGnw=J zaQ4ixIpf@2s`gabImFII=A3=^=CV62GR2dyugwH|nQq-!rJ)tDl2}gb3K@}oLDF@% z9W1$2=`y!yHJ9)yG%WQxQp+M7Go8&Y!l1TvjH_(=C4}nqC3VhgC26s*eqwyV@Rhuu zD`OG`0)^K$sq7f=ck=mc?-80AOQ(@fZkJlkBEsN4*8bRLKidtTh?0t$_ND_|zHI`K z()K}Mq|IAPlJpb8B>w++oQt?}qkT1n@q+XA9?*u!-d_rQGJse2r`Wb7L;K~LjtXv> z2@W*z`WpyUD5wb=n#l93rj4ys`yMasvKoK>=M+%7_0%7UIpr?jFjVykYW;3=?Pnt| z4`>c5XJ?XIUca@|!-<_9J=Op>o#43eC^oxU>G?Qp=#iOX!6%)Z9XdTcrB~m{5_D9V zEaT&7hUw2JC>36KgRXfcORL&6Oxjt+;`n{d!d%6ob&Z_^2pb1OGS+u2nS_E~oZi`1 z`5bPZU~&Eo$hxHv5_g6GwL!^!0az8Z2dPjC7@bT4aY9JV#W+Am@{~|an1$D z)hezrky(@FePU&$yO4eYPo(B6OtyXtHFPG`Cbxg~m~sM4|OlDDD2 zw$HvoXz5s)(N9WQr^p@u88KcpD90iN8At;MnR* z$=0tFZ+kGb>R{9zvE1c_z&!h>eS%%Q>G|~Fa!}Nvxt%%Z0!&8ZcG&3&WG3_Y5mXiy z(};MaDr#i0Z}fc~V!*%|8*w*p!yDclziMp_=3@c zOD6^EI0Z454~mzSw*LXBeIow@J|b%-i4i36ixH?!V0o+jx5v+@qn7|l)Pk5Z*NXxe zzJGYUvv&-104oRxmvoJs*d5_gkmDwqGaUh4+rGm#jT!k%E6XXYGkNha_WO=TPIo2y zd!MGborWu18UakhP$D;?hOaM~x4HxN`NVH5(+@xL4~a7Rd9Uk?BfyL8s8f$gGNLrh zoT=vwDM26(px!Qhe}Ef@ZN)C*A7HSMPuaWkCjnE7!}okBjWK_;-Z@Uc1}u&y`*FK& z_|roedul?bpkh%yteVEBqtaM{Q0z0FVU>{;DIJA!C)7erDPyg*-|8e#jU1Y2R)69& zhbT8kwgOwNX)EV3E3w4P>xrkeg2=&?D%=g{ZK-1|Ox=1-CW=8s`90v@sd&bQHtQas*T}U7VmA?ye0NX*>rJW8{e|F<3j;%;( zzi9CJFYDR@eNaVC?YM)ZtVex(DIscPL&>G`OnKDo9CK3UZ}yp&57H4OO?ja^&@2@x{WFaFrjN3as z9Fr|>GwYe2cT{}A{0`UMgMLS`zcS1bHjT}zC_ zdWq+3b1tZjdKFtheW<*XM$d#+x|ut6r86@x_x=;V&aU524%kon zV;WLO2Qn&Z7py80H={i#PKu^ZcG0hn!>9Gkw>+j#%(1Ckr6y?s+@ATP<@275*!f*F zBjg9}Cx%{AcF6ja{j)?SS3J=?hV9K94{%mW&P z5lmL;Mv?C1+X(lSbegYlN6Lo$rd0K(INBLIu=gvxsg&Y2%8})G4MY*skD5LIYUu$p z2H=eS7MiEwhAup#hOhdx+aes~-@gB*YAB%G%GmXxGh^4fT4OtKoNMK$BB@2p0>O+# zAF4`~atV>m6;0QMPjaFTSQ8R}V;=NFsbfl!!w;saF?3=2;6qM=%sst4y`=?I;sU7x zanoet=^m5K!++J|x-;T_i2&XCM+8ncz-#SQmgC0E$x7s~=j7zmc3KBTtSK^T^t4V)dX#iU$ZuQ3PSx9%TB2{1PdE9{fJhu=p4P^s)7j`ek#8^>0c zol=&{uM*7>?S~3eDvy<}2S%Pas!}K3X}cI#@v9I=9*G#z&yJ#Chd~Y8(~7hbHSe}R z*3SAHsKWRsYsB>_MXbI5Wy!5Slhk37^Z@r=oo$s+3^@u+v9#lgKke0plrVq!eCeC0 z0$MP`Fs`uw(yyTeZ_~=TY-|AU!F1GeE(#3oO-WJj>E!#o`*3C-&)PRWTnRCO0+j{U zs_VD%XQ0OZbhY?23_+j#^{ng^wUv7SE^7|S`qz)$x)L}AjC$o-LZef=m_fUh@s4NI z5$WUQ?(!8K%mf3(FP)iRJjXf%lA@K1n-W7G}2we5ctJydIcYY-*X=3?o-p`Huo`=!C zn0c^b7;4bxHq77ATixmROPFOL8WQ8lX)y^~$w{kX43CWvES{xQUCP=QO?!G~+)l`j zA(}FcAMNv;*7U9U(JUNd@^k#N_-~Muj6RnsTK|MWjUIUy7TtTlylU)zxJq>=d-H{& z1R_VXh=pL;U3Q-4tA%~cXCTd$c<3=K<*7KPvYHv&_(0uRyQmdwt z1TxycvM>}Z$GIhw;uPUX)o4J6Z#8<3(J|~X+drVg*y`vms)URpF592eJB{)$Q6AWS z9Gp+5J?#M_bg}orJ9zVAlUNBisO%?9fv5u@cT6VVOfb7DG=cpYDF z!Z|+=?_|+NlpzO!!QLh5JUU0fmN4beq5E^FJkKkZ-GNmHV;f^$Pf`jUf6pozm3Tym zoQ#oadDj|HNb3V5owr%!e|mZBVC~P3 zv^sGQ|Hs+&p7!%VJh^7uG>Vko!T)cRZss2>`}Q5x?ngH>{=eZpCYX73WRWDlg-v-1 z#>lCajfx>9W`Qi{Rocr ztAuW04$C6X{{}Zt17nJcg=OZ`s2i^mc6#~6=kthhj5_8})!}z8O$D*ra^zB4x{=g%e4GxC=uryHjv@SvS@K+yH6Wmfo#`%c zlS`muGPL#9VcEi-iBO7zsAJ$Z8HAz+(CrTFZ=plbKU$KI|tOn9sSYQqM0&_=4!uFu}WA$s+ zouLO90MoO>9kV!$c&D52K(m8e!w6k<9BF%ixa(2#%#5Px3Xn4WqeU8FrP$SP8n*Oo z?z#m*zM4~eg4DJl&VgQln#L9kq^g@MVcI`U;!7P~09Lo@oME2cX*% z`EW`Ud~t_e=G0@`FDu@kU1BPi)Nc@ z#<2nj+;C7JyNbniaIZVk>{6*?&je6xII0%=J)YyazBSDPO!<($IHd_17xu4>&{I36pIu5H8%MR z_Zy%vpE9Re7M@dit>hss8JrPdWz ziKmuHHmjn#s40BjZpL1WXTvY!NEAXgZZBF!$DFj(ZNU<}ix8DWEukYYc2VwbzpI=P zBeE8yJxXSqMr}S>4i$20F{p%iJIg>)Q)-3fx5E(yO}j{E8U56e8)$z8y28&EuXRD> z+gi-XG^s6+#=I78U3R8viyMjO-ISgKFvjZd-cVRGk>{eT;}o!Nv>2p{^lw8eY#PI_ zRg}y&3+LM;nS!9N0visQCK9nGX@>glb2;2dKZR)%bIBL2RAl98!C z>1F8jDa*CZ&K^pB*8=o9T=J^-adNFns5;3=0JDFCn>pGtOVveltvyrNs~p*U8wJJF zdZp+ju9jA6%1(y;nK2pifzQoamB$he)m>Ykr)`iJLqxL6K)u%cLQK8T{%RT?afEBd z!vk_8#tLU~q*UOl8O+mYZRbw^*vr0D?~5R|T!E13W$3hW5FJZUcQXaVGP?x{B_#4E zSBb>~^9p_G3WVsm(S86etd7uI%`adM@h@l;{Igl*uwptmU_ zG9>}Or$e_FV{69$N!x`Z4RuqWvw#HSK<{HNOH0Q{e*a8hWn{=goG?6kPye)nbmI?w6HMV0HKl-WNl5mj{z1WA&XCMFk z;z#n7>Ew7pZqnNrNH7X42r-nNorzg!q21`**Xb%hvk(DHKZ_A>IS%-z1_D2US#!TB zJ=*E(ju(sz%xW)rZ~bk8htp+OtXF?D`dxT$WM|IA`^9W#vGK(J>EnJ@Js0!Eol>c_0UZU2z6<+>og1ujj7_JNeP~w7V z`N=mSkrGiEN-989D&*^9;YHKE!*hT&wjapoPnr+5+s75F-uqlD`6p~z@^}aRLS9wz zLrg}oW`tEJnY8d$o|Y_jjWtcPs+9Bhn_?M}_OC5xP;n4Rr48bx+!;15dAS^#(2JFy z%gf$J-oOeWD(R*-3AR#xMEJ`H*mkgY7m%AJE1=;fX|92t zf5tTTC0{55nrM)b)=JOfWs6h(Dg3QMGIq=scEP@X5$AOWW~SiE2YU}T@u9$au_$>5)?^wu4#goC0SSRp6(p!5vR8k2S3>ek-~mavduf<~)A z{k?~J22EUI{>J2hx$RNhf(3jari~bgJuYjPj3bc!^LPkC8d^B$LKI&2%O^esAuU8- zhu5cdM6VHRK>o1I&7_^!VNNQ#YA8S^kv*}9ZDwXCWml@A8&j+_Gygi*t4o|zFE&eQaCnXuVSfW5C z@^+YF#2)1ISrkI9aSXWkF0O=JgUmHvaPM|4-TrI(`80_tur#Vg1sSa!nP6(LE&lS* ziLONBzv>1_x$6ho5P>ZiZj(&ekISlWFBVbEN-6kLiiXiuK5qfK6DbR+ljj+~&%QLc z70>2I73re<$ie>DKI~VqLtP|0t=iX09st}9G7uw zwBM>Jh?g%o5^ps+Qj;c#4=bqf4Klwy$4|KoMAqVt=q_w|SK?T1Ov7^?wIVQ{x@d1>G9SUmao3qdnCvGUz^^Jpz2?gv#nfc4~fPY$Qdf z=Kh+hKl}k^j&;BvY(Q$~n;N@1%(!6}7&Bw>Lb(8nlW+i@TN&_(7)~L~p;H@EKY71` z12pHc*|5ViT`&9*$d*x@Rdh%Kx|=?uumx4_aR4#LmP}#46zt)kF^%&+cVIIqASLU4 z3L;9^Wtfh;49 z^u^4XU(SrYO3cV_Jn#5Jv6bFap+Dcz!X9s5Q#l?O~w^}T?#lu;(lABsW& zaS`zt@9oLi#reQ%)uos;?Yc%u6AH1tKk=n4AN)Lk_vVQT=1oNIk+lwk@NnCgTsYpY zn_>D&Cag#D-%L7>gPi;|?N(c=R09)dgg9)3t(I#OPlLT>57h=_V*Dr~%UCVq0*z{Z z8Eo}$f4k_iM~HS1C~o5?J*Y3W$Eh$j&IbUT#;`7H3Crv7a7KeGGcsP4?$)bvB@o&5 zN*9(K`xC8Ack!4FJ7=-MJHg7`K_oqYx*|+z$<^~iL+z>~+&Ec{m;A)A#KuFU_9#qemz|K3g1T^Rc9eI&6c+It2c#V>Pvcy@LS~i zI2AZGrU5j5o$UCT&e$GF&+QVEoFiXvl49!0XnOE%b*R}vhYzoWyEX;PhVP_%*wSF2 zO(RhM%w7&#d`K9wUZ_!w*<{O{d!KE{gL_20t*UXd3ELRXRW->i)Y>Qn2?k*C_e$R7KKNtt|hcY_TpK<7k#DOo3xVm%cKqie?mg z@QQ*u@NLO}&;(iDc>6d#_GMZDvG@OaLfMl)tRoQPS7aHOv`;ptp)xa4#)0dGt-Fp| zp@jQWAUuA4{;P3wRT?#kcx`XhNK7ex9gsc`SaJ)W#JGihtLt?l-PBk{C!QtBns>!* zKwX`YG7oB=dw&d8;_{cWZB|OV`+9bV#v7AWkk3IYIk{%Ko4dd5l#}NpmtWp3u0zW6 z^3puOmEj@HdAY8`hEDGcDZQs0ETxjEosuOpx5zc(<}6R6Z6u_Hd1dTlqx?WAzJNlR z_^Yd3|11Vg?d19o5jz}oDyW<(JdTxzabsKZB{gQSPhQthN}Ye-iKzGBupsH;;x>Ws4)S&t3+v)qjRkPFAk8Pv{03e}=G@Q0BJU>Ie zQ1+Xp*~X`K8=EhyLr^+OXVXmRK8RH6LIIY`*kbKNki_(0^?rJ2!LOpvpqckjk5h59 zePX0jop9~l+{PGulz3E7y+HOUO5#^&IfJ@*CkPeND1UH}jajEO6QyR$T&0?L1-9*J zI|55PpSYIa>J7MjrwQuSy-OfFndNrZi?0Imzlj^MPVx0o1%dLgCYeB0JHHbNINgZI z#aK9y-SWI8QQu|pccM?%N1QBDz|l@?%5yJ6(6^*ypMX?^=+`^OynSEr`A^Ld*9~IG zUAL4)g|;t+_CAd->h=gBu$0^gVh78~Jz)CDa-nHl3Y3RAD1Jf*F%^mSbs+EHs2r;3 zC#*1e#==Xyi-EHz(I;?Q2He^_YnSJlHR(ODx^d>qw0s41#>ciyC~5i_>g_rrJHDZ> zuZ82zt4p@VwYzhUl0;GWA5(5RDY>~T(H2zaqovqv$ZX50F6Om;T3T_=RvOLO5oD>n z&jsd^3nv%E)bRvIH;r~XbSiUC#dZ=$a<_rm(+i+>S6|<^tGm6sismey=Zrt9w(4|r zb6&do2=S1Td167_%Q;ez4-%4f{ni*1sznm7H21~EAD|rQMrXc>5;1$l8Y1(=U7$w! zW6{V{5~>Rf+KL&dF$;^9&=GcJDnS$$7VUj}ZMom4XI+iOt4g>X8*W8izjNugA=UkB z#b2B|3!P5at({o=V2XayaKUfGp%M&5rXjcX$M3=2Ealo!4Un?VmeomjAv< zeGFPmmS)ojN$?%hszQ&x4qwmY&1lD31-g}$PdcXAEx;Wtr_nh`d9E`7wIz}W$pdVk z+S-j49+Hl%yR~(;Yp|Boppe@h!aDxGiCSKu-&n=RD_7|R`2)?!&m{-)Iu;?XX3%nt z&K{ef-FUK?;~D0}_M64@CHg=s?z(C6bGo%j!%dG$#l7{wSbB{X2TTDqOp|Y8T=)Sc zHtrp|4^`G}89zWdG-Am?yOGx9r)Hb^)Gp9MI))CxSd!2mUm_D7lOc-=L3y|_>+if< zXlp6-n(>Frd|7NtJb!Rv5ZL?luC4;;yMaeP3Z@KBR5S8l7_($45Zd)uirH}m{+G0t zF1?JtQu0F!#u1Mc$9xcJW~vEB12y%QEEB$yHDZIK$;fI}`;o1?%be8h{PE69nzNkm zz~z~e9B)?d^>~~OSs-*RV&c#Ek_@W1UqC!5g6z=qe$wMSHsHJ9v<1e)$E`FUaG6-N z{^IlB0N#{o$rK=QNDmt7cBGK$_fCdl(#p2Tqq686m29Tr)pcq}BsBD;1r`H9fELBy zT0<}9+P@l9?m%hLU%UdcQXmmxF@{ppBbMDoti7IJA6|DfWUDMPd|qlwen~al^L?{_ zzMepDJb&C|z}xbAJ?_2@yoRhHe(s$_(8ai5XFtm^xmxWYD0HH9l8ae9pZbjSK#z?0 zCl_H?n(g_ev>59bf)(;A{LVFLA^>Sk5>a1lo%0G~OiaK@4CS1!jka6lKH&jb*t(ef z@XinuP0S9GUh^HmRMx^Nz?41DArP5@*hfmDUI^%qwUt|Q3Ki5MR>}m76Z{_cJKuhI zwicoiV(0ztsUlzfg}sXIb9D5q@oN>>Oycv42SOrB%8yOqr&+zFbG{I?RKvbHOTu%Y zd|uS~>*($PUL?>X+Sd6=4&T|F?p|hR@JnO}VINu{7VbK`FO_wjJs}IHpJyd1nTB37 zVsxdkf%fVg5M#Vkqi+$5(qFvP2H2PV0Rjk;U*SR;p+?5ZxX%L)S;e0OQD5YwS@TswX{E~zXnyzBR2GgD;Cdhf#VMmK^R#do0iGvCix zaa?k!xjtHpqCN(1!OdfEkC=j&I_ySOzU}8qZPb(Klen#t^^7E^RDJi3fmR&)S*!53 zI&WOy$)Ehp;f^kNY7UIn+{g2%>yfCeqrh?-f9eMJ)GXPRN4lLYlDf)=P~`@}?R?;S z0}uFI!K=pdwPP-*1=d9xjd>a{+-sw(TocZ~ZfN~$tshbSZSv-~b-Gt4Ydi$Y@DXKnSL-QPTwhHS9eSArI_$CYQ@a7_UGUD-EC;3coU(WVz3#A=a% z({u6l=k;vw*qSN?0tcB2YbOZ;w0z)INI~@V&-Qv!;$4fPs)-c%=Ja~oomKn8%d?K- zE&u6t{+a2)SJRo}cx`(6t%K*yFdMo!{UzXf)fDCa7yy7`n|liv?y8jI7jOWZtQ@Db zF}N{3J4ylOEPWajZtpS<=^X0ZGeXHX85epHACsBdoZ1QBT~4^a{+4JcZ(* zcEHu*`z5p29cAS#t(i`+?pl9o5wiE5?0wl-L6wjpF5jF+GBBVXD!4v*@w=mGD|i$%$e5|dRpV)(x>{b`Ab?iWehSEO4@9C zbqO90^@8v^*^<7Khc9@@!cR#OI;KqpY`gZ!Hn%C~i+R-G;|pUHJ7}4MH|Q9<3`>kT zm+MX>&@EPPB>$&lLYmkRs80NI)Jc$xe2;~D!+V>LMb3T))mskSqRrhW+Y21yjpb|W zI!B?YC9~yz)9KjroY5pXW^ZZ>x>bXJ`57{6cRvV>OJ2q{$HYiiZtQcje!BUX*3?oc zno9GvpIKR{IBhV}l%ghX8{tUyXGNmWnLD`0wfcQ;nIZw2h{SO}^879vtID7XG|T(E zJR64=V1Cam|4bHvtA@r9W~czosNx9Kl!ts}3dRKrep9kKHgeJTe!q03$@m{8!{3N$ zLC5Z6_zLBye7&j{$p(VJ#GG2Oi_MqpF7oIu1u=7FU^#}tDMF24&>6V5RGQcq$RNZM zj=uZMbd99H-HMA(I%1lv@wy3gWN~NfBIhUyte3EfrtdWo>+^I3t8hAuce*{Tn||Jl zYz%m-EC047l0*wlZeK|?;YT<}K#ND9sPoHZF-;JHL=PaA@NI!T$m^a=6#tXg+MADv zySE2bR$Ss(pASQ5NV}?Vf_>yBI+6E;e;O|7oGwotC~u-{PM(5Rs>|YhMQfrNOf~+6 zZh_cDT_?dGU}A_a<#W~x$Xs){`g3A?v&HZ!Hq)3r!_S;u@~7?R6D`1f8;&hsM>+Q1pGxeV z^;!JSasOiZ;wYUe*)9&7< zuagSl43|pLp5XSbKrO~)+d)gsGIV`15R~FO;$n*!8iH5OyVE$x zK9O_`Tna%TwZx3Kle*)uoNfVxYUr1xnNgYo*SL><)(qvmv2?U0lDj=MeOoU~V5XDM zU_$ukHiQOCrSww8wTirJi+L83T_!ijP@MJwefyWH$#oH(>=>%e0IS^?ChR^`aqZm4 zRA)V@fWmaI8Eq1U$-X!tsaLmkwe;9A4YSWa&=$SVxj+|<)$x8MXaa|M$%u2bM3FuM zlr!q2k-p~+(eTi8w9`SP$`wNGSckZc@`{`gr4m~6gW`GR2EqiSeYL1=L#p8?1B?V? zUwsp-8+=5-;q*B5d3n*iQM{k$Pw8beIfsw0+{7CA$cZ0gIjKnnnFt$M{*+PS?U~+t zn~3fe6`ck6qUd*;@6+mXKjJ6kayTaYK*38wL zAAK~?sa1{DcZNGe&y>#}Udy7&JzyxHFai1@6FeJGxZe2=aWGeW8R)X{^aa~T=6klR z{Lt$2es>A&lwRDtZ$UAZKS?R$5B-)!5IAxYT>pTP#QeFJy_5%(mI^zjEZL z20);Z&=iggF4#kdCEJlKB;v?q?1SxleF4a_U#8Ef_ca#}7s4DXlB3F~ErY0m0Gb3i zrOjIY{+-Sor8Bf5EB`R4ehyw)$uW%1C|qJ?7|z&~)os0Q0RP}e=CKj$DgphcjNXzO$gL+?8(82uR5;yRhsi~e- zth#nJABJ_Gp(sK^*`J$rdfl$S5MQE*ud@(xJWNd>E8FNWzu8KcSWGZ~Typ4gG@1pi zwOmP|iI;j5`oZ*}JE|n(Z)yuEd%>p#D<7@z3We*lGk=bXvtWb-@*^CG$q7|azEur~ zymAtSTZJRTpir&aFs2in%nu|XNLY#wZNGhAWbL*lCkE$PBAVXre5jy)Z?e$tx}bQw z%5H)SP=;%sz|o>csHUWH3aIK3g(h@wcDeIr;UXiKS4kXaEdCNPo6VqB*KZDr$Lw6J z_&h+~A0aePTU|5?p~`9P^c-O4+zn_ECXHPKSG;S69fMVTxdP2sWE7U=z5RM9^lkq0 z*F$2N1gK)7ln}`DBn;j-eEPA4;C&KTi%hO2{~*^q|DAkJj@#pOKd5f%@ntw6y) zCBS}3MHfHbkeu3<*>`r+A>QB**|s(h1LyXji)Xf;Czpka!yTDsr*A+-N$F4LB|3)` zrTdrV8GfScqawbVt5$z@;S{Z<%Uh~5Lk}KTXrHQ|!gF1no_GDHr@cE)F<8#u;H5$^ zEGNT-GL(qLU|IYf|2>Sbp+pphZ8zQcO|n<^o9slJk|7ey$u?M&1WZ0qCe)Xk^7krd zxInAiU^Aic#-|1m)DDjXV2@+-7n~qoXI;74x0y;ioS zhPAc3gZ^9xhyV*I5W?7sjVj}DKLQB$Y=Av4^%evYCf77AEi&R^`wR>3gyAMK&Ey#4Sl_)2)>MAe$zxt;Ek^=~ zu8mM?Tmp=SU9D>|9IJD{My$l(AQy3)XXAGSaDgck0%sv$B=MkPO}>CW?SUD%!XdAA zj*XyTzxhG-k{CVrewIR7d_NKx==&ct&j%0ZU=^9P&8!5*rIEm z>$ppX7}n)72Y1==Tj~SwFZuo6atKzA!9PYMoB~iduBu63ZlkIW4|q&oVnykWoH^1; z(0aJmWO2$hpKOP+Q9i@gpaU~47TNQfuq9^`b#wWX$<-!;joWfYe+^G3`$0CxCv}ly zI)qPJnxdY?H^`rm79nb0GOi7y z;PCJ7oUl`s+tn;`2+e2AWX(@MnZ@S~)nDdmdC=X|c8m*%ZH_qn9U#2ofNPjF5cs!tQGwZR6m@^Db_X;UtEMyAv;_*I|G45lc^LX5 zBexMA;!!Yn4p^aRg;k-byaTI4&z{mj?fFuf)?GBX$_f-8k^$QoI1)xkh3{J95Z9!$ zKmTF@1I&SX=p$RB>I69HkO?}@Gz-L7RxDSj%Bi@fnm_bbnHV~ebMUi-?wWz7oJmk=d}d91TV=GVGGh_5 zPl$z^WhW?6sjPwR{t0j>wb!0xfgyiN7tmu*tmvhl3iab-=OUS0m{uc%h|GgUqrO7651x`_n7 zCEozuhucH7xsc5`;=%BiiAfTjB?EGugw;*mR(OJiGXyG!|6w32F*^ajF^%m|t;0nU zm!>8ps+)K4Z)61LvdbN&c4x$A|1x9tCS{K5TtW{9mrC}KcU_+}On zNvdRwoog7HQjAwkdPoWnp0y)yOk|Lsdk5+Pm1<;$)sJNCBrT%?VrpTWxf|I4xi}oL zxe>qCVu*tX=o!lgrZFJ6cleZmm|t7$q8q2P9q4kVudO@NO=uBOX(mQH-g^$!F8fx$ z$H>00^&k8J_5X`Mg2s#8vaQQIRm4#|^7)>LaT09N3zcqJ`Y4Rxn93mFo(4n(=Z4>h z?fSJOh8!E~6|IVUR~w@5r_3|$?Vkfsjx01{ zHOj1P+TZD9#?OkIse_tKwqJden)zGf`#NFjo0e|QDVCZSu`scqTiwkScu<e8$$S zU0gWC-}HKDr#mh_kQKx3`2<9@DbLI zc5i}AiEygyO&BF-9SbMPNb8F?D--;agjO$4?U3K#&k*S?JuXjqHC4{imie>Ut0Z*O z_708_!cZzyl_!}d%!v-6QsPkh8jkcf2FJDK@ywJR-3(=`96~1=AH(CO&{9nFG3zcM zq-w+zluUWHkzZ4fUBIer9h#33N<xQ*j54Zobo?Sx$o9+0qmJx` zd#FS?P^-y2_+z<&fM{KvB9!XxUpk0k%R@QZTHb6l!8Zr5s@KN2MLvq_+y!XL&BI>W zBC7aM)C<{&QA{H&kikTg_A_T7=vZGU)o#e}h%oi)9~d-V4x;c);l^hhE$xd@}pQE+rn!Mk-+U8&i!jh}**`C4pDwMYBh* zo-j)kW*aOZZ(|vFv!NiS9PfV=$N$)m<^Pr)rq2KM!|Q*;keJbO=C{%hhPP6U;gbTa z|7AZ|Mt(`9|Lopl_)GZ)@%Y4_HOU@RlZn1|Vel>JB|q-guvhy0>?8}ibb_Xp8~yyX z%v-t&Q}FYY{MiOj`KFQe&U4w@5cGr)h5>}mXn7|`oGnE!KqTL6&@YNd;q+_o z1uvK#*tDyb$vb^V2H{aQl{d}b24Zp4MiLe>WR zKP$74RZ>U-xjo4Xe1iOZ87fSb79D)&Tokf1FBk51s)Boy3MjYC4;8KrOfkU*M`Gm0 zmf3x9diSv=!|(W|35ihf_TgqST;lD{zSm6@T}^I_JHL(G!u?Zah>;YChFY(B~h+$;FWgs$&Z`-*x}y4 zPr#8B4ob$No=PXif-;^k&&Kx2aaZDU;c+g2XrM3{zy#H-J$;O@syPa!>=>PT9DLQPe zZLKij!Oi`)$_5#UD2GH>%aW_KxOCoCmZ1d>T`mzg%1(FGy2SivL5V*PI|2KsvYg0=?x+`aBWhm&)_mufQz(o=11B%N-{itzSY2m$oY0;3`$oom_^hyb64k9Wk- z)gIZf1)q_kRfmHn)KL*^={Zb{`Ng7S4P>FEr%ibb`}_WT{(AQ_V9^Me_y#=|e*G-e zVlnb$*d^G?WG)*V)VmF|5_~#-IZCC=o|1jQPENN-01DQE{sI|IU!SnjeS zs#W8wpz;{;FY^YIImx%+ARl2q^I(GI$rj4s*drNEUN|Q{ItULrlGoLj$fr z;0d-4Mdi!wfaOd*(_VfFxVyGu?pvvE7hOd+sx+i+` zAelzvJ{v$CR&)?%lnZu297^&SSgTywUaxR#oX45IIMQ-Jhvj_#UVC4;YPGhK#rdLC z<7`MKc2$0je{e76;VDU3cbV6y!;LB;zL)AC6NoFNY!u5+$0t!HJ?{8Me*zPW71mOV zM?Makj2J}<<9hP`ZWd$cS0TktUS5mM-k%eZGCSQ7A{cMMpBr@aiI7f|3cpAekJ7Df z6&_BIh(C-@tjv9_b@&6PA6w9f7jmZ(t|py?FM8oaH6C8X_l21OhV<}Dlw2HE0kz=v z#Igifm)S0G&-85bIR9@Fd69G}zHWndXdW`CXnP1KfkcNe@CL7p&gsPf>P%XtSvQXr z-Ag+{@w~UqSNn!2Qb$x)}wyhL1aKp|KP_7jS>wiAYlYi3*eiSNA!l`Mr-!MeuWxVu5WLonC#QMHh17t^Cq6tq~W< z>m-h>M;g?mOa-c^3YJ|Bq-TTW>KJBeW9`z1r|sAl5nF!fwvu_P#iiE+RibiAo2B;n zA&N7$_r%dusIEhR*e7OAW1QGx&UDhh2Ht~10~Er8JkbC>*o>i2l#>kzS%!yZX^_#t$np7l-E&z%sq2%QcSZPOMX^jm!2ROa@_6#jZ z(8#@+>XFn*ukdRs4B95{wHWE=^xmW*w980h6H)2Lkne6wl@0jtc56*JH#ho-E;dCi zgIZ&u%@S-{B5bO`=!}FTg1m8nV$L2s+IS5WL*Q2|&fJ5$`NYf}P7jH~r2Vc05;O3M+=6&viZII^RkAFVu?ya+&1#-IaFR44*+eyU z0xMS7{K+^ts>r;8(Rfji-w00vO*Kt?fS^FjH_Dd#8?Q`a+7dbIFjf(t+yTak&E5w` zo}SPCU9R3Hm>>}MMsQqb6GN)ed+88&r!3lfMRWy6_Gc+R>E3xt@o8~=aF_koZ-T|~ zWc*;^>YJ+_R1>+grWxBE@}liY+gU#icPrdM8u|-)K`Ph4Qoh_bdsN8kG2737e^JAr zeLxO7#Vn785FxqRozYBO<9ifFrF;3?NGMx(QD;vFr!nd(wLpioFiTJu{X@42RS3FP zPcyw1l*5)Xi|aazvlsul43~LDb{b-;JgRgBn^1Uqov$|wB$mT;Ng}IMDGV*xZeTPW z6{`m;v031BrD!!-VpRZ1_FUqiaSw8ek!-Q5EU=EoT}CqdGQANMS$1rinihznxo3`Eb_-P%EYQdF;M=FkFn_|)m2n?2sG z3+fOgHmPoq%7{1NeQ?WU!s6H>K5jlrq=`z6hnoyzT1*HGJEcXO=%VUFGN`uQ4Fowr zP=E2t$uqX1qgr=v-{YakPV}U=oeWP};5l0hI$dbeod~FHpPRkEv-v3^&?xJas>x2K zVYnvg$GhQTQ%H{D)6j$x4Pa4h@@GfZa!|QGeeF32FxE{m6XkEUs?%5FJwJgYe-UO` z!+Zfv(fv5F4a?SF%9Z+&Wjm#hE42h|BIpvhppadZ%u#=h8k1YuUlFj}o2+#SRFP@T z@$u?G(3+yW76LUdvCQA&fL?u7 z!={0l@JnkQd|JY5$&BNoccPr%7(Dv5Rl7A;=_rUpQ=T#tQ9x?l?MZ>n3(9aIdec%c zgcsHKNMMkqFaI|4_pEYsrXolO~cy)~#}5rMKCo!!~CQy*WEHg_Q|Dpui?R{~?5 z1<(&3-QW)i2jRqfTP5FMwPrmFwD{hNaM==i?*psHtTAyW{pB%Q9`x&?X7O?IqYPz2 zWVI`}A)gD-^^z*~F{6hICe1>_bJqZu1<=yg%o7){;c3ZV1^W!%hh1f~y>ZV@s|!cc zVJzOkZUN&a787LD&<_|Z^-u*Rp3Z%^B8x%gHv>b(%^W=)^(n6U_aMR3GKZ}l`cq)Y z&k181@hB^J?mhtHSg&m(t3Wc!FY=ANTG)K0NNcm9cWTgjrYiGIAz>(GHBX1A-C2jc z#<|f32PiI--4P4JEJuTvl5D4|0WPY;)_Y@UZJ*H2@Y7K5Ka01tMOel%Y{L<@|7^HT zJp(JY5e8M%aoP!{;13mB_jt$N-Slx3BegQPz`rX;JYo>9FMw$^oX=z?=r`ka>S@le z46XpSyuOrl7)kyT<{XAHW?qr00(-;x5e(N$i8#j@u7qQ8dXqaHbMo)^_h;DO>nHH@A0h71;0 za-aKz-K0-n^{)rMc%|QZ3*n{S{62p^_h}D+j6a>JN7zn*6L@dj-u$dNex1Ab5-P}) z3=9-C|1J@X9A#ZT{cF$A`iWSO38#%dz+V8)f!j_SOc2tiRw6hvL_d3=RuVYy^L>X= zGT;}X?^bD;3FrBShUJ@d=hDLwt7_*iOUZ`d zWX6TVF~8Huz-@^ik!zT1d^>pCpBpJSSCkXxPX4ypUg=&uX6rkL=Uc=~u)RvClu>_E zrnqApl1LPE4r)Jg!)6;D^+wN9i4s{-Vz7r8sS3kMN!SJ8=amFa7`n9wy&J8bstRe) zu8-V;Q|$;D5&_g;OJRirW|v&)a^Qo$Ye#(7Eoa(O+F$+BD9dYR;CpBFAxk)EQ@>bR z&AxQRU7k{@a8v4t4z z8;9~7Rad?)!t!R8^qe0mKcx;J!Bm-qaJ z2*hbm4I%Imz%uf4`k;H9K1?Pl1;Y+e7frc7nt`MIo>kr_z%9Eo5!j?QceqbC7Hqrr zG!!xNugoVtq=Kc0RJU3WC}I*`>Y4@Lah8&UF^*#9Pc@h;ZN&*Xj9qwF&F2p(s3@J$ z^WI6)aM~|bni%usNHNw%&_I-jp`i~eCs^%G;dz#t($=Ccc}@YK#WYNGE7?7#hGF_> zrr`?-5s{`tUQC*Ya7Kj@y6{9nGV2iNy;4myZTFjr-UQrBD;d&Cti+70z6%F!kP>Ag z1pPt^p8()3c__6IbhIK1p?43tQa4j{dYwY#0`dzm@#ENv3;&6=p~Tk)qF%m9ZD~|M zT9Lf-mCgS2a=3doz32PfhcD?^=wasDQRgdJp20pp-h5ME?@1}~<@mJSugMM(>6h9G ze^lj$4zGOb!1Y5_VWgp7FZzoj{h@1%3lew!HkhUs_=s-3R*V?Rv8I>ClE~iDxIPJ? zS&4ptF?(S#lmPNT%EH$e`ij*EK3t9POl(W zU)~_KG7~55uE(_Of1Uj{(u3SoT@A2b1Qxpew-NK;8Vwbv5JLy^zCSim{SGP)B1)^> z$PYB|mK%4$9U6mHvl2*KT0gW2T3)?Lt2UED(js(#0y9aj=Z+d!xNz?$@oS2i(Rxop zepyLdvMZ6ii?qmCVmimC!qW2Wu`4*;sa`@@TYF}4BMrlV$1Ts@pIMlJscHrT1@C;& z$e@Y~lU~mA3R9p_Cx4EVSaVe)toi-|D+*rouAL=#bOw+0!>MIJU2Zvb9!(hHk=4{;LNMT{C#p~M4?Qi|?Y zyw+-as>19-BR>OvZI?cOXMta(&F7Xuybml`D;c8p=D!k+qF7*}<+v~wRn9F6zOvY_ zzf#NMYspM(%T-T)S;P_Rrzg!uW3X4ZYboWb3(+i`oVRCTqt%XzVwV_}W5OClo`jup zQZL~_h?H)&wJJm5b$FIXxJ;s6XIf4m`SFjHKtv;I%@ zKnt{dcs}e;kHXT;W0twUq3eP^%Ab=-RqZfXx$G%3`K3_r-WS&v*AZawKUbb50}d?lf^_`AzFrYuTlvVgj;$8&8jf&9K+ z#~OVM%nG$*Ua&BhS@s7%!R;MMy8*odaGzGh zez`xvl8R1CWVbxjAS@NL8_kvtCQTlxSUiKqG?j}k%_yiArZwHgm|q0h^zv^r8$(pR zLSTmR#7YalXFB!aQk-%OL%M=vFU^S)#lm(;NR32o230C03+vk7T|yd>E;1{jE8tUW zhWz)HpM6wKd+I~uGP4R=v3GOuM{kA~u1)C1#Ca-l{)Nud6XYb#=FN2uPn8kcRQU~K z8|V>q-8hx@<~89l>G-kl;K9{Nqp0r2gj!hh5TuTqHojDAvt0Q95U!uT(@2(dmt` z#mkQ2nI^xMyVgf8I%9Y>M2g*ZGu8(5ReO46`1N2@MVX=*jz`4DuW`SZ!C*qeW@N+)0~^@OrX`1Qz|38xB8H8>8C|9fn*@*wFq=j3k_Jl6 zF+mx1eiIWV3L%w|$3qy$4N^rO=nOD-amy>5z)8DeCO<^_{6#+B?~ zy00^72~Sz(OB;6Z170wS4TAj<{FtP|4-R6fkqQMdsBe;Oei~c;(j8uw!J^s&r77b( zbkEF-7_{|_$#fQVvR)}t;({)&t$M?*NF6bfpjSb$r<4X@S}*Q-zgd9WB$#1K(q)Xx`rFz~)wR;nLisWl(&G94u@%x{v7%Bb z!TQ@-_Z~?;zJwF(C2gmQP$Rr zilVLqpsu%@r<(;y!6|rhZ9m1xgCrvGp-PVUWtw6mdkMvv9w{W)0|RfWYd%-;(0XA< zT4xvxl)yX-6OHW&4N=%=3uXnznI{z@J>9z)k49LH*Q=s;7}(AdNAUV%p7~;K%!skg zM-yXd{xpD2oGZZaqa}yjbht>V84}KFj|G&OavNT{i%GfTZl1Mt*i61ajZp+WvD5VA z>L>JoqY+Y+p#8|%Wl=N9BP>St1degNi_rpHbvRU$ z!sjHdIrcr4E@_0*-!$s^gs~A1(ufD6xNT3;h(_0PWcM?^1mSfbAHzn9X+mkyLpAML z3&X`!Ra5+~w1epg|HjwYOB3R*2+n8=soQ$$Wa^eeIgln{T!6tfC32{!_K%NSCRs9n zXCyVUXnM#>BP6xw9C5uz8D~P-Pg5kd%VRc?j6Dz)6&3J^N^zN`%;A#2Q_KSZ@{b3v?E+BjB{*MJBQ%JYwLs@g;Ub`FnfFJzOmw$3PW;14I2i zQ3m3|)7pnK%Qj8{ei(%mu>EdJ_VRu|e5{NpGR^g(XN&w&O1YrU(t9KwGm4kO>-0%5 z+Pkm}SZYu;3|M+!to2B*6TY)|du@tw965O$ZR`0kq(0u3Tybe|U;`H={qCK#ksm2> zu2=wp7SSoYM&tZK%iE6Gr9*#Y+#kcgbASI^@e@OzJ3^28F|-nnjlw}yZn)Ei6cEz^iEM^V?ACTC5vu?2J&6wZ3Cl_piI zTX5#!?vXcPqJ?YVJDyfofKp`^I5pv`(pcsN+TrW1huYObvG424>Whr?`_$^o_b9^m zqc5q)I9pevIYPo~5Pj;1PfFnf7HEX+>4+Cfy`ZkWXm6xmSQh{OT`?18HUV(GAvggL zD;v^i`%&t_B%f_yM#whPTN?i!d9;)El^uZoT9B39tJ}kyXYx!Q@JRi43g-L9(vmQa zOFbx6Ob!s`e8d2vl#g=ceZdGQ$4F>}F2>=9+^^#Z`pxDLvBS!IimDTc&{)?j*|%KD zc+@h^H3MN{{HxY2oAxX`iztzGcZlUsEI}`~j;CV`w9wGA%ahhYCKUR~iy40VSphd> z*nPFU^l5kby1V6>E%1^iL{}qF(YDh1EK8TKvZ%FRcjpFBaiZbsZuEco=3{5jZhwIt zRd^xhaw3oNapQ4m_L$x62f51hes{0QsGldq+_)YQW!5iWnvQF@!n#1|r;aY5^aIS; zMzTuxelIh<^6~02vc2;8$bQ8@zU=T^LB9C@ZGwFHmGd*|;MYsT*^PeDy?hTcNo!3g zEQh`fz+fHae6A^ig}3LUHczsmRgdd7VV-hB{1tn*@|8mR!d&vqUE^pSv-tYFZ3xRd z&NWUD=BZwza&^{1jP)9CW6nq_c$W0lD^*Z>PeRqlR)`2fnFoHHHJO|wWMh^VPohcj zdK8VQK37j@PSLDMrC2cck!9oZ<={3Jo8Lxa8T}WM^L&3l( zdie!hF%GbE0Wak{|4Xd{_KbTh))M;o_pKIkUaM&SpokrILp{pXxxjGc2T67qw2Dr& z{ok&0-_y+M9e_ev;!GBm!WbqJXERZ2OrPCthE}k7TfD!}&G^jd&VcTm63aCc@u=d1 z956EB=moZ7_hD&hbZ$fD%WVV)mT0v)#rZlW3bwb|OgR_#Ml~@*qLlFUGAj?Jmd)8@q|3BtUZU6JmG5H zZKkiR1y?UQ)w0j9K{(=EYC&me_>{#BoL5oaHQWCDIQOrcAi5`N*oNYR_$5jq+OJLl zmTkskKs(=1@P|#TwD^bSVCBG(@$oi&EZ57{it(Al8CE4I?#eTt~#pdQBW~88P6oXlw<4Q> zMyJce(f#!YI)8ohofY>-HmR>#}T(b&E^%v##Y(m?YQ^YdtJ zS!vIi^48mjh{?F3}l$%#`Hz+1WZt zTpOITLm`?9qVk>wqOzj*^MECkP3VRCuPc~Drv#}6c;@H0cTYnymks-&2`$&p2bUQV zodOSKNd!)Il$e95K6;qi^)Sjkt@Nu@s))--8&GJ5Lu9f1Yp_h0FbhE$#T@}h+yo?w zpDi5L*))TI62ZTZJ!VXUOe&SW+2+PgoXz9TC_L5|Z{J@aK?>jy2CN6dunktf&df!q zNGnRR{+xfF?>wA+h4#3hlZNk`qfqv~m8n>S?VTt zl0zX|4qb`MR2U6JIESWrQ;$BnZqQI6Q?9XCjX!GIuY6#?#RoeKLGh5Pyg@?{bo*$B zUuO6&F)!eH4XE8;+kpG4JVT4~tT2qL*!RJ==6{_9?5^8~ql{QhXL^NtoCnug|4p*( zuPH2o=(^+l7!Mworv0$8miZDnJkq|QY31O6b$(UpdZK;7YD8lYH={Kup4Azg7xY9t z&q?b*w4l4|THoyI_1qEeE-4;gTncBmuqE79)I?m+aUA-z5X?MDiD;ZY^V^;!3BSHU zo?SbUCK9R$+%|*Crv+|Vq-XG}5%IaVj$=u3Ow5PUK_#ybH%bj5-)-s^xdH+6LLn(Z zEnNA0PeH7}kQg8ug%qLQ18=To(m-CW(``2YqfQQb=1N?k zC3d2LR=8_Op!^~EP71f9cP&hdcdWUwFo|50db{X#co5Z)&S_tqg z9~49aTPIN;hKw zRjblui{KfBPTtX`Lpp5%-K?G`<7+9#PxrKn+^WwL7Tt=PGPySRhvf(TF2c4ol5rIx zAFr@hKDfABdgT~XQ?65(c4}^zr|lPsSQ|Mi{c>&E)M z5cBPJC+0l1cViye1dHYlvla-CB5GCUI%bsR&skE#kRr6-3tkM81V;U4c?a&_k52Ii z%2DFBW4UoLVL3#00K>F-oT1?{PF+=%Tp*>mz&zKa?&T1|%87?<#pnG(&`|*%=**4F z*|o$ap6p?&i}RS-lJ>@DWsAv}I&t~Xq#f3KLYh}<9BMKSul>H=6CV=Bov~F3CyS zsI81qW>)0sAYj?FE2Db~YT?L%fCB&#)ANjR>x1_KG1nyP(mcVE392fesqeLdhI0HZ z{arPCtQ$je#~jb z51wYN;KPa6bf^tFTKz3S_Tf6*igaOg5c~Xt+C_v8BtfN;R{sLA@VfMT^cecjin+wq zz7v+2v@4#+H2D2m^MeP)+ zrh$~!ZI??i=|mAM#B+*G(|{5G=k}os)Qj`2%^yEE`}_FO{2`7%J2yA`^DE0;UthQD z4V?e$?M5_bcdz#k-y660L)@5{oczxL(ir#Tbd(j{1n0~iX}J_Qp8hKN2ihCdx6^pX znFd?1QcpE?6nA?I(d1B0hXHYShi;sx9CU}-LrJ5jG{IlE4}IXYzPg2fRbQFrS3!st zUZt|eKEEZho}CQ8Uz^a|5=}R}s%#r@W3= z+L)WB8!z?Yx@%Qa=aV4j+B45l?%T}9$bhV*i*lcXSRdDY1lOy;_YbS_l};YN8IMz` z!Ei%&4jkv+wJDORDXxS+8`?71AWp zXozv1VzCTP3sZ9rqiV0qfuuOm)bag0lV4@#;6X9;4qLuwiv|D#I7|+|ujPbiE5M~@ zogT|-0S~LtzoP-3;3A&XDwG!lfc~CcfbV}4x6qET2?y4)ZY9tM+`$^c1&FC49i)m~ zjf^Ex4Q<4AI)80xoJmKzI!-0KM2emf9(m_GU8ye|WsfE78eOve z17bTz7Foh>JdJ?b>c|A&Cb{A*05rSP>&97s;5y4Z>)^09t&d{tdM=YmD0x+%ZnzBN z3lCj@mYyj_;8M1bbreObjBh=93cR!>z+rCAX{XhMrGqX}uz^9s*@&{|3oAn1VcaZH z^|tcgN^8bU`{J>5EGb1omQmkq1!j!Cwa;LQd%)Z{*Th?z0g!FNx9-L z6V;-@JpG^`qS;AaQTz}XzEBmwMbLq4vvWj*QjlKQc3_L~aozO4)YkZz1ATw!KywmO zOI@Ot>MAL%dRKyfl;+2ol$ylruVzousj?8*CR;^jJ2W4qjB5bGx1J%RWsUu)o2Q~{ zze;Pi-SYU4UnI{U;k$f8aTf?)hsmF)+%-Nu0CyV8LAq%tZQ5*$%%zp z$!o_+?2b*C0r#^?)>nw#!ky`A7rvS5DY>o$UQO$zR?IA2u?tBJJtjwgFn@a;uy4I-}_*KpcJks1Tjf^gRS`5!-+C zrw%ySdEkI48WHywiDZ|7 zDLNibxZZ%D8>1^!H6h-NGI$sB5T%D#xx|OhLG78+%5O^mJt2z z`wO;l$N;O)Ga0OCzjQ{lWEhq!pPU|A%P8O=QBRo869hkwW4&_anfWOgO063VJHN*j zoD-=8XO`g4jvqiuK*(Jv0tQ{g#BGc0Zhb$@lmnb6Cpb@(j65I^Lv%kO8)?8tHcgtT z-f#z~(_qexYzT+o?{yd)ve4N;JBDFy2+3}Z?vC-%XZP&TF6}I6za(jUGyt;4Dx;+^ zi*D?{Uxu^bPj7a*dbzx%cdUk$Y83-%%WTbGw1*|B1)UTfNZa8PZU;y@v%na9SM_-+ zXc1WFOmxO6N?kAt+R8WgJ6n~+H@Iay5{PW-i)Z(Jn%ThbrB38d7wp-^;FhJRA`dHy z(+FFJmuX$;QZBooU3Ret?$)^$mJTnVL;1|k1hA0xB4g4FWf%gAdcx42p>_jSW@ z3|z_0Ykv7Vi>$R7Cw<)4!GVx&g|iV__~MZai)#izb!2R6rl)1=_Q1q=>0(fimzZX` zgSP=pP47H0i{n%B>i6O#bN6%myUmzK{*Wtw{`@|?cR#w7n0sN<;@KvetrlpK)eAR*gKsDkt0s(1;G&A5AWNyrlC&f+8RiDli@Q3yP+Z{8LH7D zr?uH4Cyu|8#{@|ejBY-^yKKY7m2yU z3gpVH6O-=k$Oa(;d1;~3NWY?V&UV|pJ@8E4fJ}k*c!zQ(G1Fpx3QaGo9R=eYPh{6bK@`UTdO?6>~-ru&espt58mQCgu(Rw^mw#*xK)48#X^$Jf8l0}b zMXIn#uDL4SRki1Uxu7C7gQLEC2|RvYhNEM^3yDC>!th=-ypcLtNIILnZGl~=KG9%~ z6_|C{ByB0E+!BJ~q4?nP9(I?a6Qw}khmv6*7OEEpb8)>tOh&5NsG6r$HL&~<$`4E% zI~T02$&{3;ar8a$TlrD%{fe_bIxzc=RCN#iOjeE1O;@kITa+UP);$ktqV9M-m*Sv2 z2tbEAsP$;}CXI~T24{Zmv`>l|Q<7{U$}zu0Tk`qj&R12vC+#DEUu~8W$&DI-a!RbK z@R&C#*z~xx@`d4VN3gUi)cnRXxLqnP8GCCtQ4JU2X}muW35WI%fCC9Ej0M2>SqVg+iKUB z7zgSZORlpe`1|6V)?y!Ky+2nKpuMtDWFbE7G<1#O?m!kV72l=*miSe|YoRLDbcOifcY#$P`i`K`)Px zvQ*cI__Q5O3{R1cr+4%E16|z9PPScw5F_@n`tLj50r*6e5It9kUMG=TujpU~R3Evf z0gFl@-+q*u!vLM=Fk-M{Ii(Mb&zgW2>!U_vzu&d;pr)9uM90^Rg<&GGJSegay+(ps zxAIh|2)a`P`LdB=*q#uu)EPa1vptq|BG9Z{#@C7YF=>aA6Ep&kYimw>bp&rxVXNd8Hk4Sz2%iaM{#aJ16Ss$hd5U zbrF>V>{>Bb-OT7GnIDP7T%g2`MLJRBod?NW%p}&z3`T`kBPk}LPLpI&5`nQ)!1lQ} zS*8C`w%~ea^Pu{N>w~@Fd}GJyg6)IG?e0UL-FHLc+-C4>vFIKrmZhhC&)OEi?T$aI zGeZT1X)n<@1wB(Ca>sqPK@E=`wbdV>r%TQqnCnX<_8f7RVPfy8xa$WEPBxMN19gS{ z3K8YKtFC1$&`q}N6{DNCK#Qs&ZnDB19SS+nU8-T4geWDYRYc^@vW}mzfnP>peKk8I zfByTm8}`zO`_&kw3rAp z$tspn<67o~&IZ;vAE44l8O9sU79MPdO-Oy?by+R>!wg-6_{nUYfwy>w0A2c8` z5xy19CB3YbM^(XVf14Zoo1Roo;S3#a-Kq`gL_U!PdIWCt{WntuA;VR=a#}`wbPMa% z$SLUHyT0kpeVTKA@QKB0LZ!73P_AXbdzO^b%X;NiqW9r@llo^#6&e6X=h9^%*XlBj z$WWZ){)8aUeJ>Qf+-*XnMh5sR5=aJHD_9+V%MBlTfd)RK)qG~rMthv+WXe~-QG9YO z*X$IIa~uPDs{hJ$&+@Jl2re1xLg%0H2W@(aCqQ^3KN{G@CZ32cEj4$#jZyGg;|M#b zo2TVujlaaCw1asNSCQIU^^t4wB<5vHY__FS$FzQ23dX3j*7ex$_r%Szfz5Z zXBYsS(bf=`v5GV_ zaQsM)EQLD0>H>C%y&u}5JsjKJ!D?e*4f>}g(8ZZhDY!~qyaBSEp*s7KluVdjw};5T z%5F*rvpUKInO4YisdIFFDxlfMhb(!kuE9W4`#5iW%OR&NUE>&`u8)9*g;mx%n+aby z3cfl9Z{)>b&-2qU7?_JUx2=5@XYUO1UfDze*+8ssonJtp*Y$m+$xO zW6c-gJZWsT0${;bV&4X4ZM)z$LUJ$~lj%y?ZXFV$RtMpe#M)zJU$IlGCAxcQ zfr^RC0~S<~W6Nh)VCl?+L8o(s>|*qBA-~NWmU!y53h;k4ADesP%@8)j-o7jX>U9EI zOIf*;&+4GEVDOpgYHca$D4bcn&HC{wcX>E1iM5v(wXD#yI;mBz(YTdt5q3*(E=1;< zp(5WxMWbk`74+bMk8iNurnWR%#Luo`aZURm5Bpm}*a1Z18EwK=khIE)58kXyoxGD~ zz{i3jjIH07o2w^-i@)6Sa#V(vYA3j>$pjXeS|9#S zT?OJQfR5@rvS=8rJ(i2Er`eI3sou-LUgKfVO5Vccg(z?Xo)63TqkQK^+r#mDK za6@Au%*?c$S{?z;)GaoK@t^=Fqu`c$0Zu%P>;OGDz=?kY2QVyN2nurOXyy_uc+km< zLBp69PBaR!z-Z=q$BY7;z;zsh#<;-Ajo?wtMS?M)6A{06V10rlGyfu$v+0k&NaX_a zFH&h9FFE@kr1C$PMi-gcfQJh^E$eHs39m>Yq(?^yXam3xUfU2u3^v}=f()p9rcIVn zAndaNR1#@vvySjxYy?v7zEmXtc6a(A@jmqlQ<|vYzyLKEI~ke2Q>`D zc23>WV7-`H9$JbR^+=Vj$H>i=O%lmA!@~;&vLH5dN%i5J zU*P){Syosq#PRE4qw<)e=5*egrd0eP)Z&jS;P<`4TiU?)3qGsBuUA~=@u@)B6BJwp ztMKZn?a}D>)bDcG>fNv7(Tqr}3F=|;sf<7j(p~A@1ZT&(7?G^cUEuzG!c4on@|MZS z?siKYkobw4RfMq=Bq%kPRdqWsw8xpmjuGyN5NE97EA<}IRVEJE-?wiFyT6f^b-Q%T zNWC$bay2Nal_KFyJR-E!5--S#^|MqM>VH(lJpT;eS1%0KBeeN@F^l~-vb?Pa?QvaoAP~8DujH#D4sO+x_1RNt_VFnj%4jes_Ke(4NP;%oA zJOE^=@kuVmv`9EzB>w_7DFlRiqEg9eW9=+aBXH7l>X~D0MGo~9TPo|zoY}vfkqm>d z?+G|SfkWbr!#jmraMR(K$2VNxPkpZ29%%lYV{OZ< zX;5VFvJ45(+s9g;x61duf9eUH<#z7~$j`(1HTiG$1mBN?mzVSHgZ-xGxBBz}{?Gfv zZWLEviPO@_s2R)&lb(fT|)=)adwYlCOjt^jJ zu(C#JHN`?Qw4;+=>HA+t5oh*L?d(!*E`9EgqhJpb>G%+?_0;?V>LPR*iqu}H4}i_> z+h6}tHjc!qyveC#;?GRfsv}8zx7Zc+Xe!CzD$8LwzR>B4S!}(|5y7dKZx@OOTm%#& z0+IY{rPrVVU^e3#!w|+vadl`dQw_r`3%2hu#WiXQ7 zx4*0`$brD;4i^_Y_6hZ#Si@HCV&*(ed>!A8n%PnX>PSpRgY#9xBbw+l3Q)~v z;&CE3QW9671&o*-FQmXjLV0b4tSG7$Apsb-7JahnG0TGqGALFckiFINN@cetbrvT; z3q@t3d82^@tsN+Qr?|3blY~^{rySv6mVC;R7qt78m3~|F(2D5h{w1(!7;z^$mlwJH z1$?j1XN_{GU7T?$3}0?3Hi?H88F&hN>5{)AI@<3tUcANUoo7I$iP}F}0e? zZfh5>3im6N@rma;)>s?Mq&B)oa3?gb1){UF<}#486AXzY3WM&t#CEaRSK?EI3Qp^~tQTYEogB->V3^2AN{6zyk$TaKYYMw`H$Q(ydJd8ofrD-&3CI z7pdn0qGD6J)?D=#ML)0w0B6EGI&?Q=RMrn1_mb~++Ot_Mxi`}M_}(^0jHBn_;@F`@ zL}`U@@qv=cKj{S1s^E|X%q2$b-ui7RgV|AxJk5(Bh6&;RYa%U{1yrL#w4gV??cY4> zVMf|9cx+oNPHMNd@Y?DiBTF|#Tc>JKs%LB5Rs%I+27 z4V(8?amcEs#m554lkhBW@QGy*Jq=7k9j)i9wxlI&#fy^$^?9#R!<*K>9%2H0Zlk>Z zwO^&!p{|ax2Dlp}5o@TS&RJt^Wo{88stmVJJi72SOhhmqD0ee{{d*2T5kQ_*(th zc!uv6N+UEOYtt;BnJ|6Obz_z;g;1IAN!=u3t!EzAdk_rN`)hOr$UlDEclWlb!tm1e ze{IED0*ZlNLoF7A*#z44)yzTUTYJ~)*cfLp$Q@Z{Z5}6ZrdR@EEBiKDLEuyivB^Vq z&m|wJ4hcy>R3gb{J9`Gs6zGV$2E;Ls23a69=rxEs9uRBQn#W*LdMlCx8pc-4OU2T4 z#BuuAkMtpsM`H~uHpQ(f{xIp+P*{8^Mbtdkhx-=~xfy2j_!<`kJ6>70WC>8-UF^9x#j0FMSl%4!fLVG4ESP6s(aEoTn*&7Hg-9Xx%)FmXlMC z?LFEc^Yvr6q(&EdGQAo?foal*EvzSk!OBPeABxB}a#sH1Av7=PUd>$VA4&KaEbuPw zKs;J-N-;$O>vI?;Ie@lSir)p}4B;Qiu-1)FR2T;stlBVtt77@HLn_^txAGVZZ@7ZgI$-w8((0o3Dl>@^zbf!v*uZPgyWOs#&} z67;SKzZO2oOJ(fJVym8f0T!>(5OZ*%=hFMThShG%yLV!pEvlsk`SYD-ymTm)n0p5a zsw6DLZ~z9(hoTFIDe7^CjpxoXLt2c@!tCk@RzDG0;CG&Ou@NbwJZEcmNBbMA^p zPr?OE<|6?f{RDWJcyNc#-7R~|e3!RcirFp2r)T3>(Thj_rQfdI^0@m-R*{Y*v=~=x zsS@F@3Rv1?lRuLoh38H$^Ne-PavyB6NQ{x}v4Am>=35^t_*gr>M!>^iy^$xqM-S0wSA5{ZFVBDJd_AKQ zHeV-;)of(|xWJ}H@ApyHy3Z??p^Dny3gi61*AFNIJ*TqFPHltV7H>%jjA0*#mJV{S z2X4;keC9fLdSNpc`99fLHt?NvwzZq?DV+2$hX^*x$rN2Hv<`D(Q)bj45RgOt8(5>XQCrOIiPe=l-& z!#Y6{%>y(+mK+OZ=Aq#k1&idzrXEpJ?-yD6scP8aliLghTziz_01hB~HiYJTfeCsN zX+Ii3JpMf+;9^4BAa9Ud2m`nm{2FS-^aqLb|Nh5+2#3K3Xp#4g43ElEpx6LTjp7e? z&JTWj!(oyCjSS7oVs+1}K?MFdS4IZRAX=Cg{yr-hg&RtB&ObjVT8LHx?R*r>gtcr2 za+%NxklZ;Afl)Myd4fl?P-gapAXU#G>0KS0|K{~4EPIl%f9WKGy>$ODNO!}J-T!LQ zPmIdW_!K+X_M9Jn{z>d+Pb0n5V!H+y4AtK^(uM^9juiqJb=?I=Lj@QbwYo?oNL;jq z18uPW$A>EBz>qL(1L=*b@C*&Zy8qgbUm=vx$d#^IO-nqEa`SnOY(>NsISGLIK+1U2 zJO!*F5uCoPk&t?pQ7n5v$-K^g7oOG9^wPO}m7`#t&JI>YN+&m|>8Q2plEf!msWdkB zL5JmSr3v%;&-y6Ryc;nvEqG}yX(+4;I+LL`>iWvD+E^RoqS4K>f4$f#R>l(7)OB84MAaxg%XSIXaXgyV|16*70IHuAZlTNte`S*}(&9R_`Rr z>SNzY^A;mhOlh#xPQ!B%rGAH-^Z?SB1}YeOOZtiAL}(gESUqMmRH;&lAF0EV3)Lyn z33V>7%LS+2X$D8W0@bNF_0}#;zQ(G!QN`OI+HoK_WX85XIGYZiTYDMU`^~%*Ahs+w z-sB7QQ#D_8fTb4(e?)h2u9LAcsq*yiqQ`Tp2+TN3#&IHuLfv1aK8XN9n#M=<$AF7M zpM{D2EIaM6(~-<~@rg1J-gmNRB_s*GRumKMG%Rxl?KpmLB7QD>2(e^K8)<84hlYO# zxU8g@-X4-WQ~)4~A~1Q0E!xafoJpwdh1|e$;xP~*VtUH1OYb=a^-k~b8b(fx*t^xOA0h%c0Aa<`WUI)3wjJxgbbG^9-Hb=d{62gMLE z0Vw+zLr_(_T$E>tDHW2)j!EGMsVS%+X;skQNonGT5Lgrl40-B%-yu~t2u}>q8WG=E zX{B@p8*C*5fOxGZDUjpRBDt%ewQll)&kC2K$@i8McAC#B_fOfW z{<)R5o@*najwW1LA7aK{-OQ|P*g>+&u8k&`ouWF#vjg)5mKVQ-82eq=uDfSdy`$b4FDR;eE;1UuMup(G3 zFeTKXZe_Wo-s)mUBXD@V5$`LM2GbxX@8I8SuT zbzBP4YBQ^>^6SbM60gVqB&_w&49mYksmQ8xqC zkhJQXDVF)=O^8~&=mi%zY+Fd??zuY_LzH5|i5SW77&HR3Te+H-cmeX<+x{h(1HN1< z#tL$v7Iey6;4EF&b?p86`?c+QA?@L1U$>Otil+;%Myk71mN|_NJ_|FjxEHO9CZRU`mvdgAEbv^JTJbyswD8t*;=Z~} zz%!6|vwYMPz1<5Hp=ElJ;i0}XN?7p))x7wVeQiEH2(9fFv2-3@cLm(VKEx)-KgGsE zGAS}eM~z1iWWP?S79Q+e+@GlG*`0V|v}x^e+Bt8}n!w+sSHEL` z*2oMWi1!L~Cj=#?fjV{XJT*c`A^^|h_;{iFRPgKH`V7gx7mEc$mbNJmgN*q+roXItQ$hVEe4u3u=FnvT6I+5Mr=CnXW))YJ%&W*;G1>UZVFvDWpB>z z*T+E`wVz63_?bF>H|Hh6-jobAK+Z679o-~P$u$9mE3G>wFrWWT>w4xTE1(2oon|Y`^H%OMSALSpCX-P&$@oVuu3A zBz|3DtTFIlA2AllLs)C22}vCeL7Uxd64s8Ar8v;R%CtO6>?W`39VK!k|FF#yV22&I zJ6b5vX|Ao62~t?xW(!05n+0PD6ntKQDL97)&Pk5@UKR#Vi>ZzyEeX7b1opZGP^wDR zFoUB?bsgwK19szvq7VT`gHG)`gY_uI0a}|${R#+o@9C%*={jx_UjyiaEwM|myvJKd znN?*0xu=l$Q=}=IO*D<|pY&@v0M&+xexROU9wl@J{0gAf!s^f~ zGBTZk&NM6uz&h|wV8Y2>5z%{Ip{R;`1_J^<22I1}5uN5@)8WBrZmEzaXr5ke0g-_k zSs)|KK_ODL#up=r;BzS|46{#!o-#sh0Jrq$OA8n7G*lA=VC9a^|= zP&;)JG_KA4+;|nLr8yJ>34JMNc=4BAX?v{^?T_Vser%^c}^S{KCwzAJt8VG|U5pa-AJkYgqKgEy-(S?w0AZMNxTm0E9rKXAxokkg4+h2ZtO z_14y=jD*3YY3cOC3mL{}Qr{3w720$02v=nryjKztf+x#)WoJpIIvHkY7%jrg zr1Sj*M%jlca6W>0BdJwhq;v7&MG5kUBnrtrLgY40W#!irdGiScCFe|P;BjcIF$|bH z)fS44sARw~u_&zg#8-EYBC?GgN%61PC`hy&o8z;~F0`riXHlxZP34zH_Ivk1)u3#48t-Zg;$#OTTqZ@Kbz}`~IJou53N!FwtX2U&`UlU8OH#A?D^ z9*!NdE@~7lk?xIjQu|9Wl%9$a?Na_C&O~6;z`$QLEA-|{3s0}2i@O;q8%vzDC!~dx zOUiS8FfmFH&Yv*E2Z*Hj)H}_fBn2h1%O9q! z#GJBhU2{Gl)4*1ct|BDix)%`n>hu8Niwpt+W$fnn8rQVuHmr7fmbNO1g>Zs=3#zib+v$Swf^&0L^Fc7JU62xacl4HAAr-mq{MV*ko%s) z6OjXTS$47J38Q(x1nv9WTToh~I`|Gueb_9X-Vv*6CuqnnzydJ%?r!mK{ zKW9i2ei2hJH_*BKYA)Tq5Z5HJT1-wpFuAO`i>%yyVCg<$$};vVy*L~Bku|WVjSFui zfPOW66}Ox?I|16~niO5^08uFE)2bh4Nuwf&zO@mnRloi~z!M_Me{Q$w#$lPwESkyw zXcZ8Lt)eZ9Bx&es<7Uc=9u~fOG(De(2bc)ef!nZ}vC%o<=*zu+|1@$9%>{ooN(=<* z4ntHo)fT%iF2g>Sxp~2wJYFKt2_!CyLnb-=ail{HvlneOr*Psi0$#RK?n9eIAKhD2 z`LY6$Jx!z_#vP9G93BuVpeIuJ~cJt#6LQhx1NG^PgF2`}II=kj+ zM6DXYt7XxeXlq4!GvJyK_VPC@olb2kQk^u;mpxV3=90>|J}u?&lXxTqT?1+NxsFs0 zoa#uVuXBxl=$Ln|SOxbIdDbOdITlioHjM1RLS5G(rd*YyVA>Vv4`5!iVr!FMC;E)k z0IZzN3D=c-s+$EP)-0(0*3u--xG;Vj<6N zwCzSyv^;TiZ*#f_f6(DGaYBzvUjrN7ShsD1$=G-SIwiV()6T3k8f2CnVaR< z9v2`y$dIs8=Q83CI1kAPJ0+r{aw>&8V9uwiG(2qm6Ni9=_4PE=fPt^-SE@cF{Tb&3 z8$0up)Ptce(*5(IzPx5Vc<5pgs-{G^opNqg+>HI{+un%5yWjxKFt;@7 z0ce_62gL}MF;-!+RGg?C7}Q@GoR@(=!`H>{_wyuuFXbqobO1O=Ql1+Zbw)%*Dyx9F z-GEdCrQDKQt;VxoC6O7Aq^8nkvZ9bnH1FO-PYo3WVB9aZ1vnQ}0LBvF6t*0CM1xLR zaKbtTB}TGEA$*D|Hcmn`ThC@sItFX3-6U}U#2ydonWO5H)epz#cQ@sV>oG&J9??Pa ztG-F$i+h!Lh-(Jf#hFwkxiBdw56mASFXO)bJ$f5K3@`%K>*5;S~c)5S1FC8RA3-E3>Tc(bGoq0+ozJC4nY9LldqH5%6rdfk`H}JONayogq z^MA0m|5BJw{$};9BsVN|%yf_YmT}%dXU|KQ|g z=#5^rkDBupo6DQhq&y4Rs#TZMNCRoQO0v{c*f?}`nz?wpNlmzt7T|(6`0AE$XJk)cT&xbKvC6nC^MC~U|$2g-KM1a zf|p6|<$1F$l<-`7*}XuuAKX}FflE1&ZzB}m2)csQP;Or1M9c$Cl!x_Qbcznmx=GQY z^VPv(#lA}oqTtCuTrH}2%35YD(P2vAcAUQfGm|@0DYwk4uH(5vYD0^DQIb@Ov&*fu zy`YOhaFf-f+RGU|!v8;CW~tv?fs+(0q2=VCKHf5lU6nAg`iGJ_LHj0y?q;N>xOTEs zVO)AFmmnAz8=5(5{oZ>Lg_Hh_I5XsR4AyP>Vb!-PACPF|f z!I3Tw(M=*N9pm-_?53=V`ORBS0QD6&`^i8Lj&0Dvyz8Q|Vrn;zBAs<+7s? z6G`2MN$j34yvw+R2d`<;BE)?Ht^%^T0)G-rXQ!t2=i87GC!D{-T)TDKrv6!=9P4vk z1451tmySMkV~2c*e%$`8qDq?#@*ml6>l%7!{pCcfw6Vf5r*e#-)kK`iIOe1= z2O?n(gf8E&#BJOcAGj;J^WVfWX{Zf>9G9boh5>IChd){OUEf#d=Le0qUH4;cWCEii zGfN{>gDwuo00R9~A%{aARcEkCCfW_b#r`OusFMAKQ1i<*H8tkj{3O}%1P1kLP9`*2nA?SA*|fqhJhc6utlMsg`R7?18`P78oWeCJhg}h zKDS;NkuPc)Hh6h%TaBuP4?hT(b6;JAG!#zBE&dL@ z({`FNW)+=emb>$RsVKKpg|*M&6w!|Uy|&=&f^d#b=Q9a#x(56@vl3B*0`@lnhCbjV zCOyJ5Cs%yN_!Z%(TI(hOSYDa^fUjFf|xAH(@6J z7riwdF1!MNi3vqnlCJMUlhhsHtvx3zRSr_#nvq8*xD>lx9W2?(7O)`WOMi$f zQ9FB`hK< zz8n>w`m5h-yN~uL=hZTa_C}s#u7XP962ZQjgjgC?(L_!XsJEKnd?8NZ8s*n&BNxNy zZ+T1bf`Jo75iwe?voEm~?SwC;FP{_IDyVEp4Jh4-mz*uSmgOPHWzoBKIVBWyo!n&0 zM9V81D@v;xsBEsKC)Zdn$|&kNCD^-m%73U9CDUOmB(V+RtNLGQG`|DSFS4 zDN2fXOk+&A)?E=N@=}Z5x(T}vsj8vdF}vwERMWr>MQHTgy%D7`SQDLS$yj5Yj?MB9 z*5bP1iMMLjHQjkVS*cRMzYYl%Xx>-N(QSzQ%=gy7sjeg@$$~&oTnDlsHwl!|Jbh;| z@(3*)CbJm4G$ODlgq@w6yeJT=9F5%<7~A-T*uxn?{p`E#^8P*NebPj8efX}PYxm3M zoM`{61Qgh^!*;+U_>3jvo<3{aGwzyqKGcTPY&Z~HuG*n_ib9%ece%2w*aZqEierrr zg|%TUkvV%0RLim$q!v*-+=JmOj1?{fqoYUKZ;Mg&S1hxRHHzub?9g=jT+ognmns~U zOf`~Tb$Bc|-S~+wg1LxbhKyHg=i3ReGSJ=*ET1(pVj|2f(Cq;NgjG~%hek0hDlUss zIjm=22HSlC``oD@Z3EETvu$F0rq>i*_VwuT_`d9)JJsHd$4??K8lt)S9a zl?LCj*`WZ1v+-=jF1}KZd05zs*LL!(5*IrL+2ctgNh#v`zIU~}_{X`Iwy+fL_zPup zEp(-i`*i2sV`Z<_M%X@sB=MA|&=lag1L+9KTr|~HRKsynTG8TcEHNnEfV2O5bNs~| z|BsK~^K}gTiDKF#D|46?o;Pz{hZQ>=%SwPO>!ErlP}9s5xKQ7DMTw>X?3!=la^f!E z<9Ap-HFnLJ&!KXF_B%j>s2Nl_*lm)n?wL^0{o1#hVhtLnUoBxNOQU(wS|rNwRb`3e zoNYxU88%9{ORK;CjRGDeJ(cddkJs&5V=ppquPAcRKzaNxDGRbJuyK zz_q8UP;II4w%r$Y6@$!$8!8vBcm+PzKDE=?mG2vPOx?k-M_#7kcX_qeGqy&`01Pf34*wA z^7u??)cs;yaXx8fQI@Gjl1h_C)+%_AI3$+NC$>a+a%@STDYQWH zYc9jS2XsvTD;FKl!tNyS+s0JSA@2=$naICjJhx72cEjwdqE>A9Rnlz+dM1R_k+{X{=1r*Vmi)%&sgSDd$@ zqDx4aNQG4Os2LhJbgfdH-lR~PDFT3EW3t)_c#|chMER65<$W!1DTP8r;?~IC(rGlE zAhmH6hZ0d4{W(O5!kP|e7IKm$8*car)G;K>W=&IgG2??S)lJsD+i(0*W#26fj$XOy zbnHzo3Eu?$*D^#_qTE>qGL(4*4@9;m3a&awK*6@~rp48YLDD!;&*u?Hum2B{KyANh zqRkAIXe~`%p4?W5Q3KExOkR>ilvv0gL3GV2PDM_+6Ne$nUZ}>eR5&JOs9bp0l-BzY zD~V=Yf}gy*7G5pTuK0$20sA+wjA!p(`-=!hRPXMVY3Og3)g?|OYUuh5m{Q-GSItlw zo0)PR4Mo-=m{~+kHzV7sCa~3C56|ePmF#iVI+A_lLa9ef{)$@#-yVgk;}@PC(-M(l zbi~AFiq+@EjJy}sc@cv?WXa&K1(WYZ1;2UJrNhCFTsSR}>F{;MuA!n8 z7MzYbtoRJbE9V6frCy;{XpzCMh{F3 zBm#fT>c|+^=yl%bs5ltpK{?&3oAv)1quGCwE#>Y~Cit8Y$%}QyI^SvAx7~$my`!Ut zY{0Z23jb4Q6rnkkT8@4-|?cL6!^WtdM_|7`F&G1=ov*64*g71zF z6(=H2Si~#i(P7%{`NorY&<(YC*ZIZ>FO?P{TaGQ;kTb6V0_0AJl0(%V6(xv3(JFXC zv!qr+j1J>ajeD(>M#9d7xT&{hPaDrEFEAXY9WSV?!ei99VY@jhwD^9ztyl=CtJ}1( zc0SR2U5Xm+n<)`IYAKw}ngAtxmbh+`=8XY!CCVjFQ>=Ic1Mp=IEk(5| z`fp$Ww7XF(a|H1yeI=Q%2e`_s{6|ub{9pXj^E+Sy%dF%txJZ*jNp2~b;~{6ddWy# zuv@0KJ>^nY9X2r&*XrbcS4diMxdI33|9mapi^}*zoP}f{MWL4_yj8k7WICjLrMEK1K7Xk+5cCOAsd3BpSNr*b{OoK|&Ez@x z8UFu5|Njf}Gx?d23vxm1|EvFzpUGS;vYwi6HJ{JAgw)E|B~qVpV6)gIkeoz#+zo2V zFaC%57V7+0L=?J(=O-|QQ*sBE!<^orRr)NcYMQCRo_sNX?v6$fMYow-E)=6QGd+2J zy68+DiWs(2bhc}z-kCLwnif0g(oGKxDL5;3cXSsd2u5yM^IJwXe7#|%9c>H#aJa|Z z6O zCtJCTZ&Y(-3!cqPz?Jr#kfn|bqu^n(?}c7+&oNcJ1`UagBFxM+mECQ6 zrY{J*7;@dhIDhEV$Htu1k()q@{>sBdz07=oX}_3RtKb`{e*ceAJig`Seuno_Gv!Do zjsN$`W9=~=Je_|$X=i&15f6q)xXn@x@c``6*5kcs1Qf}u8VJaY6zmSTqFNm9eHvIa zofK;`)0vVRpq{(V7@X%H4ECtTZB=;1%W={r`x{mVZyPk(Cb&tc73z4$3)ruFq+Fgi@CvlNlNALH( zvv~AI+>twm%MM4+IsXgrsM~{hs{YZOE#OlPtJ6}eVL2g@T*5YgyTuSkTC$g9PW~ne zQNBc}S9K*@R^}Xr*D}S6B&$=F8uiR#%uUU)^W#^qo}*6E8W|S&`f|KQZYpJ zE0*GmUPELX^WRY=mCQ!1c%xb8oVu+M-3wwCBI4+~o5WPSLyo!o+Q3*2I_GXv6h@Q% zqUYS4rEel5auS|sYC z^rLL5Va#FmK)2Z-+PRo7ViD&6V?S`i#52^dq?Z2oAz(Y6CAk}Mp#vK^6(}atD^>m zu~w%@XG);7I`nA_N6nYr5h-!RQkPaa6R7$>vOs{}@l2)ie!t9K) zb)>beBe3F#4tah|dw^xc_uH*Pia+_{>GP-0_Y{0*qqgFI0spmDdGr7C)qtJE6WO5W z)VV2yzc_8}`s9nJFP=U>QFDHAs%CFl=3mU8pAru^Xs1OzGbTH>`SZdg zE{ihNG30s^oz1G5VIdF2*K@;f;rdxO_D%q~0ha==$dg=@?1}r(H|;oW=PuKC5DC{E zw=_5Az`?&&UUoF!C+!1o9U2SoM~~|AiZ}oJ;?0NicW>TboxOcYuIx|S5-764SK);f zV?M;!9)@>f04oL>-p4xG-&22#D;7_?QtLK&8ipz;8wU@&--@zopk|m{kjr|jeiur^ zQ<5&!Ex9J5#JVQe<11yT1{{Ak~{2|+N^-p7Ifca%-jJLBumjSbVwIDH~TgFzZ0W+|`Ob|b(+tM5|V^KHKWnaw(sr4Y9EngNi>Mm2_ zDTo?U3N@ez>IU?rO-d}poX&Z|)tXT7absCMR$26+|J}wKQ?ZUvvPcBQLU!bos8u1# z<%&G-EsOst)MDMC3abhRB6!^d9biV#uuV(>v@uZCS&IVS`@C=lR3d{*T;9H#86b6w z<_*?V&5XfMjY7G7`2Dy69Ys;MdU5Wc+|p95M4|Tvur}xpA~5O>A&6sVtHcG&-5Z@X zvIz475}ayOdOupax5WT_+3-=2y|C1)5Z)2_eCzs!A)nVv8|!HX49BUs`O5`AC#G=h z5NO{mnCcc5^64;%{Q;@TZlKg;WIO-zY-g*6Z@_qd@jNG>`G|H`1SYNZ>`YX{`59u`D z+1I;Y9A_UD%2A;l70QwAj0)wa&vQZ)N>->HRm!MV%Me{+OSklERi^OwA(cw&*qGh% z{5!Be29E0jeH=F9)KxWs7xie8AuH#Ot@k4|){6Kf=zPc4vmR;IBgxvv)vpxy_B~9_ z^~$(Ml5ac6wm@#%FVj9{d3Io!q5*UZlTbTC7WJ=&5XlrKmHH);Um68TqFtmAI)C1T z!)JYK!c5SARG~!xSn%O^n0LJ0#GF)-5?i-?waaBZA5eU=C^PVZ7Az4o2D!9_9b*x3 zBrDp>L+hi$oJvNX-p($*Zb23Bvr18_w~Mz$%!*r{vhR5oi;nUY8$jZ9oYYD@5RqU} zzfw)aVj^^VC*~h>QD&VnjOa|asp}B_P%xRBI^qCfHM{tlr1U*d0aDz~cK;5bCM}s% zkZ0t6fl30YBUIKTS2WGLAZm&|7=J zZdCwVEzRDy4m}KZt9|tkf!y-Us~>7>WByujwoFEO7uHrw$K6(d(2Zt0QMLL|yDXws zUXj;9(>gY#+zVDM6NFDS8UO7ho+(k#CDS^HS@*JqxM3(3hr;9P!Q$#u$$K%lcE{f` zg`7LJgq^Xsdew_i%&i^qwj4~Y&cO}A+G^lwb+$MFXRC>&)!AYX##Vnat4FOo7_L@t zFZ%Gb9tBHlZyc@ub43q+)+1qO9U3>QclrYGvUbGE@>(-?#L4odzU+vR#Nae_EKq@HOlqe#de25<_to20USZQK}0KiQN*Og+6^>Y zJHpd!_FS}vdiUa6Jjlz0O2wnE6A|5MY6NDow!PUHX-3YzpOFhL?=-o6KoTZZcLT1B zlDl%w5(Zggpfsd3T49#8DaXa$=R*2()Gcj>8WSb1ixabweoY@F!{N^LaP@80W%3KL z`YvR5wo@Cq@p)Sc0M1|lh6DTBT3`ZTUV+kUQIZFWIWxd%rsS6AUy;j9mRL|rZG5KJ zBnT;?RpQXRHk6qo7zAD;d=I(A9Mc}(?-TZ3eevX&JbAy6`hS^P$R|^BmF5YLxYC+R zwqQkpt7Xp23*I*Q!c|WmWl?y)kCtVvJja4OXuX-;Yl+B;LKIb0Kh_ z4iFz~5}!Fb$QTV(5eDJ&9#yZ{W$OIUP)WS?D)iN|K(7`=U{%*+Q-BtWeQ;Sl7)70ELiP`;$jOp`Z=mM^k9-nDroK%u0&!0@m%<4zS?zpSxye%$7QFR3B zBAh-UPelPeifBd>M(>%;0@93Tja8SQb*9piz^YXwsB#tA1+(|azF{%4iqhmNEkqG9 z522X>U1+O?t${zYe1Z;o#TEqjC3g;!uvj41mc#gl5RJTUnh>ZE; zb#54qiQc)#_~-WpSIohznb;ScX^v+gPGg6Y4iQ>7Z=nt_=UN0~xS!`F2K6nC`A{Pwn z6vbazh??FN2+Os#vDXNEXt-;IgQypIe~VAiOqo@$87C2DLrbQfO|5o_xG6w}y}8j~ zm8a5pwrrr}slImtw$*1-;F-O8&x#dW3X^@YQz_b0f9aRSmg9OK`1a;%FInx2Q`?r+ z`sr3#4L4@2xaSe;IWQO-)#Hil-4|2*xh^IfaXv_**RNQ42*)?TtTTacE{LOd(6|bW z!H=97UIcCHIrdej3uF<%X@-r@zF~)l3=w`No{bgQrTz_E6S(^k+s9ouam4b~A`E&! zKl*akw3wc=d_`W)Dh=E5BKqtI&~1tK)gws~zF-~z?;yBUk#QgR@78FJ>>GM^Z|q!< z*Uqn$L?TJBP>FZ-iUks)YO>|%W{S7T>D?dT@Mex_CpZBoH)*{Ogr?m?$wR zC7W>NlZ*k_gn^-rA&7pt$Gt~d%uQ;gN%+YQsrms69GG?{N)UJhT`fA7)Vwkv9Gq_f1;Y?=-1BYy94Kddy9* z{2f~pfCgQNLa)uDxt5rUfCwk0WW^Le{izVQPtZYUZ&*j^qi)jPutghX&CiuT*w1iK z18b($T6Lh6)QIr<@o5{H_7;*7_9+L@KS~JwKuOb^#k5-vlisW0i(Wsi7oc!ORAF_! zg2n=(lnR(}zm#&8GmTT8eKOpjU&pj6%g*{E&*E;XZHV2uH`?H34VQ&$2I*w9mjFgDCaj2u}3q?Sx?*`(_CL?v9S=B@80u9 zK7jW3NuK+I{6 zG`jP8MMnMLv)2#yaQNC~gNPg^*Q^c{!wvK?cjHI)+xAwzXStn^dGMZcK%*e0q9%irA-Le)%O41)(Hn5l=TC^frtw{QL9gQ}RD&uU|pI23h_1Y&JdkLQ2&$Br_N^R9kl~V8*IRX$0j- zrb;tS5QeGy%%nsZX>Xe;E$;Lg(QxD`WSz*W&P%}(s`x#6mr}ZRv$=ZvdNiFLAqx=& zk%|^(&+FfPIzWKIexu5Du9p~!c3WJHHgZX*q14AK>2%iwL4#_0ur?1`h$-z6%Pk!{ zN{v*0yQ5Ubt3@^|%vwvBD~w%%8_?`QMKWHGAeiwQBAzktIE1cECTBLht58{92dRPq zEe?^${b9~{9at{owWD;#D^y0~6(*_i>htj7)!1X!&4xfDsMu;CIG>Z)?bHoZ@B<#Ing`SJ^ z*%gTke$NU?o__61T{pE>Iz9hE<_Y*xc%U*oZdr5YY`MDII7%F1-x0O4uuS!Og9J@r zg=N?+BN6+~8G?H-OdB#-b+ zse;c-86Gts}(xhR#* zVemM-w!LE?(+k#SU^A8HH$7S2CDH2T@|o|5?2%$SIMvWn=52L!s*O&y&%>#9g-$gs zNfR!is7{05*fLapE8S4P#YN5OP`%wu`*Jk#K21iLOR~3arNwi`qg+E_vx^-08YP+~3TA5P}i$EOo8;fK>SUsvWSft&y^iFx)T$H39W%5j=~**cN0D zOdYbL8?i_KaCMBJ?OnSMk1I=T(|y=bEkbr-zw)uEV1%?9q=GcnjDTiC+=q^HFvNKj z;68LTiy-H0i2KmdFM`-akOtDwG6I_Ytlfu>(7ZOD#BEn%01;gKHqaT1Ll|{`TG#8< zTg`Y9L-uOXa33BWO{Yf~_)#AiO0}5$fv_wRaj2JC_py)))T-JPqNE%NfMrNC>{_4<6%-s@)UoZMtK>+GtAQ|OGU9h zuA0%WH`+B%JPYBM1tU>e6f9H8`Vn=migJC)?Df^0l-gYe=F>{111~mdNg*^Ojw=(< z_Op9(PYc;UVlrIKiupa4B51j2iMW0nuviJ`DMOfC^PS+%x8lNZQ-9x7TSd-n_D3N3 zO_9xdZ*1ti&;6xw?Bzf<{P^3SCpFx{(;uhs$8Vnf>DkZrAHRL}?CH}V{_^^p+w04} z@n=8(kd^5j{_E$bKd{Tc1vc{R*`L~h95%0p#>X=?5yb>a{}t4#3uwd=XqWv`3#p2U zRa*KZEcGEOc5m_4W)B`+!w!vK`Eb!Gy$ylO?1BfTFxsb^i3^IQxhQC{Cb2P%tq0V< za0+1TUGxw77xQADEXI_@yi5la(tzh(iabD(T2uVi##G>AekZOeYEGj&JSA1+L|_#s z&-Td^;|EU}?Xv}YQ;UJcb7}Xgbp+}qq`!#T zADBfG={I=OJWqH81@Yme3e#LIn_H;q`<|y|N=m60xyZ~j-d4?ysXF*zYxzTQ6rMqY z&GUPoPB5z|S(}0&qtqTRWeJB3GXm6qy)B9bxc4h{Q&ttxw>D$OlknYW3s_f^P-Z3& z?SpKePg2*}QN!sMZZ_gANV+|#9&d^_r!uq?6xTVN`I=B0$5nBE>nOghj5S8=MeE2^ zZs9@I;;|dC!NB8+O!R-OvB!G&=4^ohE8V+Bw*}{GlMWNJgek~iVO80FjGjrwypaV%(br(Xr19dz%QM9)lkvpa3g6d$!JM7iwN-)QxA zlu&7@+uE)oGuofJi*a?53*-HPeh^BotJ2HpZ#qvXPsvls7`e1d8t=^f>M~j* zi)O5tsNkbAtOe#USq**lBS5F*8!~*toGc|uiqEXR}Y9jm$HebsxKX7Pk3df8oncitB% zQ0>%4DAuiIt67D5q!qP6dz9b;VCkI2ag9BLrj&<_6?Ru#e|LUN_>@iUzr0P_W~>-c zVu>;KRmuk}co>3XZ}zsS>15k|_n&&;`prL1{?kD__~Sb*aSHuF07Qs+!;R~%z}wzk zi+Gl^Xln3z-r>QRuuK&_6}GNL?0}bJp9}VP8$AJ2R@rSMs#w+=uxztoGocKQ^#a^e znkG-dT8PCDj)@Wi_zv(wZ<}WsVeh$wN+?KMm>P<~=9Pk?KdD!OEqvo&sKP9v-(*7l zrds+oh!8XmLFm2U7S~F%fD8=wJtQ!*JYSQPOGWS4l+3ik;f5~YOlAaX^s|_hnd1KF zBmHLeeISFYq+BhCEF&!x^MXyRI@GuL+Zd`T8k7qY#jd}!q8Qng$GwAxxuKcalAE#I zFm{newJHP?+f&fy4QN2g+C&YZwZQmwG8Qo@X|X;AOf#-*Of*K+`unP9Af*|ihZQUm znN&p?!AwyyFT@?os?-E64Q_o(AhA&aI5Rea6+$-ziB>dQvO1MJr7^6Lzf>oHK5K@Z z4Z6@8Hdin$bFGk69s;232Kft&pAx}C54;kR6NSUHBRdi5f_G?-pXPJl>Aea*g z6PB}CodiVAU5IAj7b{C6sHLQ)OusLFt{hBeBnuOYK!F^Nmi#%pHU=MxOJ z%xdj!5Jhk6#Q&&{vISbS>6-n{OWh#L6fc;qYoHgnA#|DH!fukh-!QFt_L3=C2WO%@ z(~Qw*FUE2M!VadPw^eG473~KDoJ1*%T+-6FZ}3B0bchG{TEt=gybb8K`sF(L0+V|r zn78nOAh$ybt*A|mTe70}yIY!T?R8Qi%|MqX@VUwJ5SK_mM4`O~dfGzrRqdsD)GXL7 zv{Ct|de^Xe6Sq-C1_i)V*^6<_g1#cZX(%{G8Lm$Bn2__-9`UDULTv4*=Hnp_YqIe{ z(9GITsU~qK%$aC4f31lZ2t036CiQ|YM3S%q$YoCdVauRV#RL#n(Dij(nSZLIO2!_7 z+O+x^l~_&E#wCE+%SI*IZ-zMz>wY0XGPH4)t&x})&Z0qsPf)8%JuH3X$>==M0^M1B zRezguFN~(bY3YV7jHSl?so8!|tF`QHw2#8K(fG6qWDRQAX!cIMNR!T1$HTgIs<03S zwbd)P5>D$#0wOww#&kd=EyRO`J-xw=&aSWEWqU4M!auCE4|WpUYtA$~s4FG`q1|r` z4$;G_xrhG(xE;pvRzb8EOn)i3Nf_xAU;=5jHbHu7#fx~7(?YG$Msr;CKDc2wZ8Gqs z!7(9yP(tq2m~SAR^_UqqYZaRSvll$hdX^t+lp#5q_G*mMh2F3hmrrz*;V~Ofy@PwI z8@mtorqH{c4GiwsFMygfdF4KOlV-KD83+|R0@46Z7j6<g@dxGCiqf>MKsQ7boW4uSVDmVsyHIH9i!IfDp=XkQP>u(raweav+D7DGtDlU}gE&6(;$9R(QNg{=uuxlEl7Z%t&F zmhc<$J^0X_me9PKB;r95^&l#%7ixcD)3q?%NFHw-2TZ$Kz0^|~_2GzDEkbk0dapmQ zB+>u(aqJ$`w`cTxnPGN6m)10Cl53vEg~;V0e15SP8Sr0dPcrA5b%|J zsq(Tr>;qv+5ZVyXK7gJZVl~d%1+6QSdsg5AxO|_}OmdOEq&x}r2-&4lJB0wMt{I;&&$g&Kxb=B&Qr%L>`mhAKnoO%aEmhB2g=s!}#< ztwa#ea6lJ#HTRF9abhw?sJI$lLay<4cxEiwF*}y*mLW?%)#Qxkb%114vc-E|VMaUt zI5)>an^{Etc=(%_Sbz<$RLn`4H#}kRmS|L6JV|LrmuylsJE@a{yy(mBHPBf>s#N@O zfBbUy$BixXu(3l>GKWuYl~=gELmEYL?e~W0_{!y}GO|)KUvqC~+cp{NW9-G`1v%;#zA5NuMJeNZ5A^Q8t`TVn(Qy>+q{M@PP~HPtK6##$_nQOuxP@7 zg4_#BteYu;402625k7dbi`@$^J7;^4p_=wa67U^c_f&M=zObs5$mELW7QPquxmgY5 zJDTtztH>Q89(J#?WAa8Q{l81dAQ`wW^>rbbd?OUR|3M}Rhrh=Z8UG#!zZzzFL8Xm> z{iyp=#Zp$2&n3Cah$swoch~}^lh{4$UZ}m{16yqApP6t`G-mK%U9EiJJ*ht17q$cT zQ-~m`(n0{tk1Dnd2()TmDJzyx9WDxjzpAf+G77I9*Y&XQdwUHrFXGh?fYp53E}C!+ zdFUZ^_{c#KdIgD2ALeAc!<=qwZrEYeI#QZzuJeDjaseCY{|z|D@#oLeBst5ylB+`cDwZ2jbTlP?9g_~a^k zQrG`%@SsYEgP4XVa4)cYF)^81n=0@y)%G#ZP5XwR8awTILeZTv+}6L@LJLq<5S?aD zJGHaSWX^6Gp@W*-(@+g1H^P#z1~G(9NRd5=-`SLtu^XJg0^x2eX0oL=8g&wx1GlA_ zvuzM&7k{fxHVSG}6eeMMD?AzSziO10emuVVs>3u2CUb!yx~)VQpuwyF(%OdYDUz2( zE~LSI+T1f?oRe1-jcs};W-V761X4&%+Hz2fXU!UI#=!)Ir;Kk|v=btzbsI{G$F>UV z2D^%hGvittn7-&+LXeF)*x?3r#0eF~FV-9RKBl6$Q@SeBI8uf<6-qhBFOcI>xzc zkKr7W9cLnRDY<62ZKa}$9U~s%BC)V3!9pk%F<+Pn!kOa?J;q`ZrXGY_ggK~LOWwmd zQStvZ!;6AvVl$etJ&tnsHR_3^d?ZQQhp=R?7F;i=Zf^^qHA-y;6p*nO@6=bm1AZzN zNcPzK+|cp-%6M|l4K>MA}NhlJgZ7%W>q=up8SbibgC*LX6u2_v_}S;)-2B&+!eQd z4O#oTugnX;scpVrqwX0t0Hnd=EC?rf*1u@G!E(t}TKOz_=E!AcUhuZd8`{m(fMu@4 z1BnHBU=WC;^;F0#kRgaU&gda2FA`~9h?LtXdAla|F2I|VUPz?o3@2r#r?*C$d3jD0 zEtX8_c1TvHbLK5-dRUCH!O)#Y6?SK16uSd&5^`f(gbXCmAs3>EY>20Y3Ps>JJ@elg z*;r;udWCivs3(&D`|sa`gRLFa?{v-PgD8^tfyezt-XwXd8+zs&Nq}j05E884c=_;e9$()v!rvL)%6|g3Fz|z>3wAl z{gNpx8U{rlgjx~F(=t&sW1^JF+Pj=SWtg?q>{W<(LDB^MW)zFo|H@8SsLqu;^EiO-lc>h-~Op|knr~v5xYUuf1YZFhwYMNN z^Rup?qr;(;f%r341`y)DCqsF|!(d-&Qlas4*a_1nv?I)Z6PF)$Cd(uT?66Q+RV4t) zyDG&A%-HNK*(`K{21>plnLlu`*}FzIueHy$4#2@`mGf2nB296;S8UL0osN^^r ze`n1|y)XfRdp;zmuDLSaIKs!~Qw_GViF;PubM|mzGQK7cT&*T1S@{GM$dg~g|9Y(r zc&IvyW0SLq^-@iBhN%PbdJL&O$E4)(pZcs~gH)bgrWah^g~4!bz1G}X z-<)3(^K0PArQJ^0XDIA$R@4>a_r$bWMyz(F#9ZF(1MRiD>2dSCV38+kXc+H$TZ@uB zm;&SGoN~+augGO4%YxOQKpUUwTf>2;7IKNT=H*xReBy0*pb2x+OzCa=(5u1wu^@Zt zrcJc(l<_$B(wr6wj$7f2vwFp{Lg$BU-_?1wun9d)KRB1wL;gc{RYOioA$JA7tZ-S# zNzCp~WK5qQLl;7_b(`a)l013-WJ+c{yn={GG>xRF@-s-Vv#;dFlG1c2@NqOTIb3oRV+NQO1a6 zSvRZ?hjpnV-4`DLCDbtVoTb3P)4MkRGm7xxjeggeK}3z zeOMIB;#}~|U!q|HN87$)*N*+cLGm*7m;5?YMMouak%(nG2J-+y{2GIiCv>7Iy#KLA zt%jUv)XEa19$BT$t5t9|!ABqXftA8dYV`Rx<_tF$-~+0agTj@wBITI-)6f?EXQWO| z^6?0~uM$~pv2Hc98->4@4Qh^b?OEK-xwTI>nP6V9NrgVM;0c53-B#9i`);$^)MG7k z){cj7C+m$+K1UbpY!*!DT$-A-NwThi0F8BoE)f}~FrJdU;P*UXOP?Wd#j~YTMa!+t zW$M*64f(J2zv?;{oejhOc=PrafP2ATy!Syzcf_+K2$@VKU{ZSecdawW$usgjGv9~w z237D#k%yJ2U{Yah{WA(%74hE8+b6mOt$(6;Y_!9*{6k z4S`Cse4Y}yP7|Kp)gVR-VTvR&qHoysTQKU+FW8h0d#!-IxxX4@Wd?(?-e-TdY{CS4 zxQF>CPhfOC8v92cdBD+r51z|qBzYLag-09QHX~QF^VwC?FS29&bU*;L_ki5x7wHnF zFq*^Ys?mz(XQf(Qa0x{i!xfaRH=P^|L<%U;mGbZn?Tk!^rc~^_=2J5^~I0FIAuQGiq=_}majDJ2R z8GA5&;`V*=pT?{RXdlTRzY2ygub~Pp;;XS0MIy^s)CZayHYNWEXM6k~*@mSQhx*?o zO^|$V4q@hNl0Uq6jn`~P>uGpX*-J{<6YWJaGcr}sOzL@S`b&(g}PzEoICROw%j9VxMnpIxxr)r%WtfSlWYiLE1Km7g_G`Zis`2DGO@;_|e z$zV&;2lX!QtYcVM)M=M(xa%_V=H;y6 zsoV0%bz4w2J$3sYow7wI1s=)X>C~TY98xt=035EEKBjHXGhp(O+);YuNR2+v(Qrpp zzhbd>a9t2Z`xTdBNCxH+K5umvy?>DT6q@~O#?Pb-slB$j>OShq+RNZOG2_^k&F5{U zh{8Br>#L@OkgcLr^FPnFG8t^Ppix_GBpXLVQwL3}dwn0S-q2>5DJZ|P7xRP`hDC=~ zuWTkCcaW>^K(b9rLZy|uCVb7!Qhp51yf>}Mqw(#EZGYdJCF&(g>$&eRXTE-(QLUMs zIm1dQk?oLK)AnN1X@3n*oc9-Ot5*cZFITQ4z!^3&QDoMs)O0EBu}6PNY4wLR>F>K$ z^LRian||2R{OZDJm^KqlnngweRvFn|X^$?^N1mgD8d+>1e*P5kX__Z6=X+ zpGH2dy*Gosw}H3UwFmtgadl`NxCQ-wkjn42u?0V^t+%_B?5WC7iB0clcB;y-_xS^t z`~aR~UoeIYJ#jV1p5!;)cIY}Y@4y8iFn23EEjrRyrl_YvrMCQU^nZgR) z;r0m8l+qkw;7KeVvIknk91OUQf7V&hm5rR+s`}5~h z@;_&PRWH6e&^cys9Naw%g#_qv ze=dowpw8b4I8F3G$_hz{$F48XK)dWg(swqQmEUCRcNBQNOL3;|BQ#y794 zR&g3k0hyZd&_rZQn6)1Nh;W0ASX^cCuJay`O*m#$gwffzMJd(SWfms^g<~6|eR96l z0><{5)C>z}9aMi<&(XUnRK`_xd+%yU+wG49v}#P}dOy*O&{ByBl%co5)>HChMxd_p zG1NT{DP(RmsO|nZv+veY4ykqfW7g7M)qB1DjH767QME$5n$eUUYi?@`95<^4O1{ik zf%cd%&?vfr*#;#13+m8X#^_xy30*D$UF1GwmNLyK68_J+hT)2l>&w>=0}d*&R>c&{ zFu5_TCb(XJwHiDnGZ zEy{BiPYF5Kn{mM-s)E1__1iIvORZO%9MKgcqKFxm$6t`hY0$`^!0iQ+fa@O4xhiO} zM&-rMon}^uPY_69=oO+hLiUwk-6w+q?O>hF2H;C>@F#o~RJ`rYC5M380bmXa?9Ihb z&_6;YVIXq@TjX}@RdH%eP4x**l{87#6ZT%SXMWE@goV8@rPbgeFm;TJIxm>Qx;;;D z9GQTOj3@ADVAB9xP%5l?(A6-zXT{pI9n|h{fWuw!H-{i_1~*AO@RcnXoCDe&3{|uP z#@S;aaRy>>4r=j^8Ajzp;~d%|G&mHGb5}&p9WXf`36=9exSV@{V&|h^cJ6`Nxg&1p9&3F!?9N@#JBQ+T z?u6jE6Ncw5G}r;hb6+IS9cTH`&^#Xv&-0NHJs$(pGaS`(NRtNt^8mP>JM4?WFYJfz z85TE=SL>PAF2bFxW&}+~rb@LrRdyUronvfZ%X(Ebj`m^GHgLNOTW)+f4YkxRN_I?Q zksT>x-pNIb#h=PN{D|*_@M~eZJsD5sGrfGXqfqWl?zP z#P!u!XE$*2xHs-2fMRm0b`?Sd!)^;OkfQT%4kLY(JylQEr(%tB%-7Sy7|(7^p%kog zvM`1A4IpJp#{*lp>7Hd)gOoPxyoaB!aHZ56pu73?AQ^}b^4ByU=u)cQb8#F9or_Z0 z>{z@XKrX0&u7vfz9oRTM;AqlJWrfg#X6(x`VKiE46>I`ZPT-o-3a8a9FZ5t5U)iSI z89+7SELJCA!uVaP7%ORlj6z!k9W!$xfUOPA>^1u{j6YP5+eWBPxYXUfV{5rRicooC z$6BY3>+NAeJ<81a#t?b}t`?;5OJ<5IQ)}0AlGIl`3guWFfq6}fb|qAkeH>cUp}r(j zWscp25(9d)OiBb~CMhlMSX^mVp|f3JM9^yu#06cr~L#ow}n%<776-BkW8GoJMnpc}~e|Bk%?;9;$U&tN{@#S=%OSX*zh=MkIy{Ve{&60Ws9#KE&D{cpSoOM@wR;b$v%OU~dtL zz|no+QWJO$EjSU_Y)DxOygr<%(Y|V7PpUF=4NX39fTAFpN5kRAs@qt;E=>3jGkXnM z`1@GHZE*eB_0>RYg7apFJ;8b6v$ZH(p+&(7U?Lawu)Z6j{sf=scRxIBb?v1Vm316P zu-I@XR0)%^Pz&E095mc1Wb%iB(LCaT4BGE0cC>jlx%-Wke#WYb)|l{K<0Wi)nfNPk z#u1ujLSZn>F`D8BITA?~M(pNg7f$xsAl-4-SgTq->=m?P+OSlT(Tb&Xu%4x_mwINd zje1!}G84s$R1&cm9{k~EtUbAZ9vE#|Upog#EbLJBBJH(PKpJh9aKuiA9)5^Z4WP21 zX=Z&mb; z50Kin)n-tt+n^@Nd+HX>136f9H8 z`U7j`75&0PNb0LOp%tube8yitH=$zfXVs?az~Ee|q}#httV_{Oz|-f1JV}zj^kjXFuD2{Px+ir%!+Q z%j<7$uP^_`pZ)wpR;G9Oub-d(z%Kt5*vPYIfBOFhcC>LW;F+3;VuDlp6+}P|YR-~- zzkOYc{u-WT*a3mSAwjeI543T2JO1^e@^|&`qn!SRyIVdL7s`OfvFyu08#BJtVO};j zW2!<@9zQwD^2dl(!xHX+t&V|epK+iX&UC{x0P$x-`Wg_WhVf^CjfO_4fr~qM>5oxr z{nr?!Hb$w9QEEfl9HZ36D77(4ZH!V2xyC59F-q+KQEGT-Yl&409Gt=Dssq*jmEzQ- zUSJVBiy~2G>Q>ybfB=*YTf$Zg05P>`TL`fwfc0U!xyKD*qh-wXF@A?NW?yx|9kZCl z)s4U}Y6_&csTQ}{TP-*P2cvOukfW_8)*!Z{S1!{HuXMs)}{ z$^(~-$dxFQm=x^q#vdKZ@NxXNPH!4fLqOtc2*Wx`b8fI_9)nM@_c>&q@oS2*2EfLYAW3-KfR;`Rw^xl%y!e8wL|Mq`#W0_e6686& zi6$j`zE&h*R4VfPl%zZ>RcMI<=g9K=F9vMG@WaZEMGf2o3jAH2Qi3bD5;!pH^_B~c zQ|X>ojOZG^RI<=Acb`d`ec*%rs(f!l|0u%(KnXeE)5J+=376?6(6eXy5~D z^uUgnLUfg7Axdvig2q_jX?sGp9tEgqt$(zDNMPRf&J?uMJ4KHMaM6ZMVP_-+^k3|N z;OA}eK(N6DTXLz2Ar!$Env2tvN)iQ0%S7=!si@-}#EostNJ-Z_LLJjPMPwO-bj>OE zGvB9GA6TXUX82!95Dd21TzE~MF z(QL(k9<#_K8w1C#E5ImwPt!bMa(ed%u(g>zbyCdZlbg%4i`SRaG~Nefav+G-ksraX0q*RV zb($1wQIZFWbrucs-7U|*BA1yg0b+-$jnDMzX~0UtaPY{bA+BGe`~YZp`$EeWbN@LT zO{gpJKw?22Amrd)pWVQ#FReAjnao8Y{G^UVninGF(ux%W=e9QnCr{~ez;cAgE29JK zP5jAJDhso4587?NUWtc^5|dK03HQdn1r7L)tsxLFc*y33#@SB^F!I9hTG_by`r=#J zXE$=x>Jz4-lZfT36Vv+c%;>=6Nkzt$D8k{0j_eyK1tXCp1}jD^$T`bbRg;5JuMa%_ zfRK~G19FHHDHp=)lGuQ379weGEBFO>m9ub)c~3>XPh=S#7?OPxaL~`}vvz=A2Y>y8 zk*ObNJt~;$Ndeh}S~su2`P^|d(o(Hhrg(&wUD+RS=&~+ywZKzF%woX4DR|5zsrXa9 z2)#{aJoymR4RzmyIBB2N#m6F z~J(0Vk6o98Y0V01K&3nr!Ipk|eLhj?~?)LAe4pWH8rNJH#}!5QS6Xx76oWbDsQ&(9l~d(dPpVJzQm~H73U{ zXHyI?az{^6$n=8HsUjJOWSQjP1ax9q41^Dtb$$S@BrN6LdbFb+ z^x5k{QYM!fohN=zYjZ(YuaK7sT@nWWIElH`|0lEAEAXV0nK=u1sQ|tO!+)!cQe+DI z?^R@UgMh!CcL>j73kd4ek26p72CLI5{RV@L=8W8~cyxE|I8Q{Of6JVof{i!>-V0i1 zW?0ul32O_`DPShaYc3Y@4}d!?BLP8TS(KmR#gz;tN7^$h^1z0l&YG8_LPC z2|t6oi8<}rT|t$vTC%z9s)iGkJ}sF~O`U`-Z<`^7uDTMmybW&@wmL+ys-WUjCaEek z@Z|LOUwkno7rX#~AzmS9*DecMFi=J@%L&e;{cOP~@t&$wE!kB4e#ng~RQqO6?U*eU znjIAuVA$(jXvsjo@@Amh-0~iJ2IZX(%*F1^Ht)|P_qXfjm>)EId_c~)n+@)5GbB@d zI|saXy1MeidosWM{I8i6ei&4eU9m_49Bl(Lx&t${!;?CKva%1Ai#;IhHZ1QtINN$a zB}#9Obw}QGK(hxG#&|t~NXBc3V8(3c5W4o&ifraM2eddu7Kg&m9atjcwO1DNu-xS# zNdRF90KwVGLyBp7bCS37j00M1$wcnWE)Hn6G6W3DDGp|IVYvnUS;GOD2M0?g2+K4b zm|t3Fj{3Em;qV7TjOl|68AtUyt>zh9H>y{?F!7jmNp&SO4}9d)F$r6z{Wb*rs_0ZT z+U#_1XB$q`>SwEH8y{)66Ls{R9>RBe$Y|1!=f#VEnO;2EsI6&@alkx4O%wAFgbd)m z)&4uXz9Lbe(1_ds{WyqSfiBQY_7TaBxQ_-e>Q`JBUMIRi-qX=p77h zJV&Z}DX=L-ro`-36s=(SgOs*11FMt!yd8yE`1x4z;zThV(i$^u(ff;=&|o@iOQ z7XI?y)-qjV{sT2-%bZoKA;8ES3u~C00M7qbbJcX&a`}&}G2Y8u%i8a}DQ3+r;J(q+ z9PejvYgfI|R5`#5PMb{fF?2`2-q>fM{YC4ImG3{{(faCQc+3F4XQhv<;sx~Y z*x+lL(Ipxgp3bhne*H|Z6tEs$U2J-wU2O|L@MizC#tjk-lD=DSJ^~pIkhy8)S06N0 zi{!lCELefT0k$@dv@9e-d0=zWRi()t-t2Hvi z0WLkz0}<*~w==N~O$=Mbq4ayEHbzCh%_LJ=kj@jD-TB3M;JHvg5UJaOW>RYowx>G` zb_Oj?s-~(@o8k7IW49)8bGYO>VN^1b(mSo`^zd#c>ARE+y?Iz=Lu=enJA+F*6X;I@ z3s6s^^FlmGScZZmg2wLVv@Rk%-P2qvOSC^I1k?3VGX)u}>yM7Qj3XwCKVdkp3Pw9j+cKn(I6DF48wNxUK0gh_GQiOH@A&}J^rL#u&mE< zIMY)F<37mL|8+L2Q-5}v`@ha+6DND9o_v}lyU+bTlh*9f{$vWU!0PV42vmzO1p@ZQ z$qRe%pOQC1nNlX^4wqp5^r--q(E#M*?W@_8e7|Cu{tGom8|GDqlOLPrJchq=$0~S3->z;NGzDlj+BWB z0<$L1B9V44%lfnJbnA~pNqMHn^%XhwkhD8AC8tYv%T>~0yJ(Q}7E?3avygtBDE)qW z|9f_(O61@(k)F)kSF_Dykw+`*KXgQ_@KjlAbi+U-*8R}$@&zOcRoRw(g?xjLK?}Zo zO}&*Bx4Vb$I8gmnd1~vT>@OIqXt88|PwV3bJl@ukjtDz<&_Ktq%KD$SH}g?y<%iYD zA3*8w`uHsXmd#+LdTC>&`+(u>QVT!44*sFy$bw7L?pjAb1cGlvFNCNDKMVPEm_#%9 zt6~{Y*lbEdai}2@2+EP$mhPChCIDU|+xgdKJ6q@)0pt0_^PGU@Bc88VF)O+9rwY28 z!P=thBNMhlBs2q@N$X_yxBz{t87olhho4VhKtw%DbG1Gu--yb4(QKR+{ZaB~_mn4a zFx-AO7Ezh9%*mf!it0z&cXGGed?^x|E#dr}-z`s~Vx22-q7r%X>-=sxi9{w9&6EdU zyEj2*z;W6avK%FxW$LGh?xIgA(e zKI-Ko+v&bWz5KJ`IU#yEs99&FTKR)2o7-Uqnia`0z`{}2nu^#I(ye{Ery%E52QUnd zGdU$MApsU#hl=t9%68PxiGHtVfc|{m73zS zb`uCnAeZMCf4=mMmS(YB(K~jmU&yPD5K+Xen7Zc_{800N;%T|-=n_O={jIBukAF0| zW9wOu?Cp`e*7~o-++nT$z2f$fMC}e@Hjwf6i`P$Cv_3-CP$1pHgynES31YB+HG~Ms zP(-Od;pi8PK|*mCftb?wX7U0V!E571c8pAQ*)#C^YNlFOPV!5-+gG#azw2|DdzVSd zvw;IZckuxS@M5n4plCwv;e5D7@LXi75XnY`M$|=+1Dd#A5ba04;iN(;)16|&uy{-w z9R5gdJnXX-In7{c)}mx}4p-G+EuxPJ1;EmRNq_Lak;EX;HM=x57XabKHLf1(x%C?^ z`?s-w3s1l**Kw;7wypOAV&GeDcHEGPs&ZrPG-b%Pwmw7edj*iI}P0 zC%K3vrhB9MQz5RVJkzvV8-(DtR01?l*c>U|i1v?Jo`|(x#@mb49}D9PbkSYm9Vdh> zZ7M1Yt#H7iU@dFH*Tqi#b(3jfcif~p%>HpcSum`JDf`N&RvAupCy&Wl!U~l_a!`R~ zW@#qmO_{KwYPpRc^y#Pxl^CrK1GwdZxrE!V+2kOA5;jBDPnp27ZdQese0P>gX}jI2 zuiCYvo1S>UPJ1O*%B1vQRy79w=VnH=dDWH`2J0OhW@WGOzgoaY5{aZLT6T@NcO;hk zeZw+S^gC!!Z?|6CT*cFrDH>Bnr_1V@?&-9sZ`BaV12y=NRG)XtTx59s=iu4W0+KJt zySA^q>poaYyJC_xdH&*<%uD?2yD6@qY0V>k_}g@wHr}TH<9K5TToO@&r_uyT0=TQ@ z+G=9~cQ)JL1$O9t%iD)r-Cv}tU{nUKf*W|$MHe{u`fQWs9le<~K%LGPbl2xr>&164O?&N&$kKZ-BJ=aj$a9v3eI46Y~3=29beyW z0za72O_b-HYJIjxnBUZF$({~qwk8y#d~LQ*nZ>L~06ZVXWt)&SW&^O2;q2zD(3_6M zQ(}tYBuufY@v3}i^;icn+jZ_@H~lOV*+j7-`%F(MT|_RO;6hDvR=z%MfI9P z0BQjE#%8AEl13~0D`IuD^_db`rYS9IQ=fj@?C=4aXrF=?a7Jhw<=~S-r{LQ>L2REF zkZ%2=u(NVwh$!4+@U6@6L-ZOx(Orw7(djFjieEGq)Xp-q%nq}1oOXiuvord~B`}^% z;=Kmg!=yc$vRy=|G_UBrErcUE;5y|jdd%VbHIl8_Z+)up-89R$^;m-I9%<=dGR^D-cDd&|9 z&Nd8Sl=W9YIi4AX6sqJ9YfrFhYW1w{j~$VHRL>o5$@(#Pee*%nru(QlACv$4|9FJmM{_ptz-{b#JUwnc8KYjki7hjzI z+w(7e|K*G4&rhG9{{G)iUz~pN```Z?IX%n}{Z}a!E&lEFlodS6J&Mb=))BKDWznEsS5+Hm3mD~u)dloT7SDgoqqXZdTK#-+n$+3MSNl)ZF#nw zMxtOs>aYCNw(@>8nM{6FyZyf+H0SKSVi}|yR3N_3e|5*R_$zW=N+r@8!$}~E6^u5& zvIb1ePH2`145TrvCPK$%!ivd~WmBy)%uAlctboStr|(av&wn??)L%QCY2E3@vq*?v znRpDcIkZ*+{7SPZ-J@k8%KR(R{tz~4drlQw0tyiRdqP@B4tJ0~wa~jMO&NQw7X78= z<|}T_jd@ZQG}#&hvh9^9pgwrx`-N>=@@!cqwAg(4R|KniepQ!w)bh+`lIXT3hI04M zv88Cm(ki(@93G7bK1jVBdt+GORnrDt(~$vI$|pY z;iyGP**Y!5g7?gTUh@!$VSx{@*g!>S8+dkeIlHYaS8d89(%o^))Kj1d2E`){z)B%f zXp?1eE_kM#aEZPxtn>RKvZYFdsZPdeS>h0Z@IbgnhXAo1sNcY3F zZ|WYj;)8_wsx_Me1QuMEiA})MYXFLsCu_p=GKGUq#mCxBi(9;H7Y{UWf&1>LxTclM zJWOJS=P@S9N<0v;P%Jb2)kd`HQFdBiXu;}ncH$gRr8(4BaQ8j`GZ`Hew(Jx-sg*4ZP2U{vZ@COtEub5DbHT9Y^jfS&v%{AToZ{VKiS$$KS@<+&8stb zxG8F`@%0WxjUfM!NS#7FP~0zRaLh_QK+9EXpuep)6Sa;7)aozLdIEb zA5FC8#^_Ag>WPYPPOQ-@^-puw+_9qA_+n?1gv?ko=&2d&|ATGwZ%^$cK0G{Z96Vsc z>Yq+fr#oqsSUs@~{R9*^X^R)Lmng^&RsZ+?aMhCe@~^8OCh&u=C;EZz2=A~ke;D|H z$ZnUzz`G7Co(bCrYm~P|V!l@g0u*9W2Rr(Du73Q%bU)4Bx<5_N@bM=Yugu zQPAW%I72Zw0j)vNv${)A=q?6tGN7Ou0t1zRwmltyBU-O!v)SR^EHCyxZ}Et`7tPjh zy&yI=*dh#`Zn7gOHK+LRS2k+dB-~qd$xn)gKZIv$B8mxq-Z90tUyHVQfzCdj85O30 zP#wu*RsiNR9JEA+*h6bDwD-*RoN3*Wsj|?gVvK>}k|ZKq_T=1tJpB-_z}dZHEdL29 z^V2XQ|An#qCA|1j6qOl4YuuVTl{5l%p@+))K*+O_*(st6#WteNO!S~FJxF+8R?CV? z5+#g!>lBhO4Gpi)WlAxO=Tl+5uP~^{a9vSAVPz;qe-OTj0&Pd(3IWOH?4kq_rOHc% zV3~uG=u)N%9w7`vue|MP_OQ}8Bppk$^3K}<)8Nm+{E%n~Lml_6Uk~-#e;DuN$M(-W z_Ozy=;Oh zXzgOTEZ7o^?KCQclq4+^#d8DrWU%XaX8dSwE~9Jyn2sL9(PP-4{^&9M5FSH^5zKNH znW@|KBf4>ZjcvO@UyKYuhS9aZi9aK_R4Y;Ne|F^mH4P~4vkpQ;DH#zNVfU<9+s>F7 z9}1G?U?8_2PByBtV1hrj6;P~)>9DQu+*Kq{h=O?W05WK7<7nwfsg$d6CHrjkcLKC!4P)bULl_sNf$ zEs*k+exA{S6+d?10(n=BhkW#(Wx-|_C@lJdKh7EtJ36Kf*nkJXuD~O}go*SHYn#)C z92t^X*4-h5p(X`W1!wmFgo|RrRH_8F)=E6=_IIyA9bpyc*ume|{MWC6xH)B zG_T%7xyGV<&D2VCTGW~h^4G97Hx_LQ4IC53ffKyNx^{&Dc#Sb#tGmDfY0yVgWSVk) zKoAQ&FSBe?$`cs&361m~n8dt*5cTzm$$Xt4KC2${1fHB!+G@p+yV#=-IctK$>Abr! zMz2ZH!+uc*4a|RgYdE#7Q+Rb@thAOB)^#;}o!_c7jq|v38Xfk=U!2Z64ixxZW9b|P zuD3yB3Gs9H6tjDjQg(w2*##NR`-<3osyfgk6tD}TS4i=?$0%JFRJbmrY+XNQba%|7M{=n9i&2VX4=nlz;Bm-EC^ zulDMi&^Rubl*ed&09}}8QZX8XN6Z4++cRImz}4yXf0bQuWnrn|i-O73)sQgMo8~(; zke{?1Lf@`JTf1eAA`dGbHKHFO@(x5bjT^;cK==C`LshNP0ddE}gkWXj0-LpQzCw(t z^eVlkXTsVk-Bxz5U{lVkJ_c$fP~gIO4b+mem#EcW6wG5Yx^9i0YzG&lN`{9mQyJoI^!RjddfTd@VknWw%OC_!p9!l znQf}gaMrF)jFz)cRcjnAXN{hJ7M8P8vZ8ynZR~sd@B#L{ZED>2?ltcFao>N4eeY>5 z5}7QMP!sZb^XV2FGCuJEjmXU|IX&v?iPu|-*I~`bNaFqM$Vnm|W{P$}aBqdNC*pxf z{k$bZ!DN{zNuEk)Y!43)Q|z4{8_oI2f>EW9oF@?eE)~n7^>n4uq@|Ldy}rl8WnAjx zq@c0ZQSqyIj?W}h$L?vgluUX-z$~^RdQ!YPzHj&SIIi0W*XF)%Oee%6*Yc~Ieb6P< zhir>}rsrVh#r_B}mG=6+TgDXb0r%R>8s71p{ovt4C1I#ykA^9>6QWpfqv7CqW^g+h!SEXS&B$-|quO!hXkuGETe3_!ZSU;M zW*MC)?9aEi*Dr!b^}StK!n{i0U60kMb70`RKaACAA(BKq+%SxM46EF8Zeum7&7wzA z=d@s(_~Y;1zwaDIVZYjAIXl!?E7MPK0V|U{Mhk{Dpz326w^l!LsaH{6aM{x{&r-_7 zbs^q|)v0>&&OH4NQ)MENrgS1%P78{)!U~KeqKZTwu3{uPxw@9e>mKrKIfZzoEN(U#(X9GD1V&+5HBX6{A2V{LEFY8` zrOxhOOuzWu^mN*Fq79&b%wDt2gr@o$Rt6$KPBj;rh*_0WrMdVC2I=RN7m&=i;0Z86 z5o7JsZYwNV@_Uw9FMxkN9yk_HXcaGv6IM|&7-IWkdg@Ens;?lMF-ANOIceKh{F<+a z?D|5U_?qUMkk~NB7Zz^yNRq3IgQ&le{oo)ir?s1~?o7l@AMT#LOrAoY&Ko)!5~k2^ zjmIkWl^WmIAu>dfB!SBKR<}Mmg`Di5lm|5$qPUZxG!IqaNr)0(FL~7CgNEg}%ma0K zGC-dvfjT`2(CbOKZclc$ClIK>;>*CW*M>80!)VaPYCbhqatQkX+0{lsM)vdzV^3Q) zhM;kcylLnsLej`cmvV@&c@@MyWzDM-niRFsEb4o3G>h&>ztJq(`K?EC!&u+(H7%&G zUiB`8CIhE^gPk>d^lmfl3QTsnzQ+_R{0mwMLi3zuv8^|x#2w4Xw>Pi6d@P9IP=_i! z=3GX5M8Y*mIF(EK+}l;LgL?H(QQ)e_?Bj`DgTo77eLe5eU<5dNrFz(rb3mY67=? zzee*Z>obp_%>>ddOp|#WJtk0#3DaRh>JIz#mmp!fi|*2NIbCmYl4=Sx`>GB_7((8? zZeaJJ!l0RxekCr!U3LEU&6~^f+scPH-SGHyHo1!@=)j&2;Il6akzNigNASdP30q$^ zG`10h-?!f8Yqy>Oxu*rE^MpzARO@p;UEjR@>;D{sZa1_4pjk}byv2Wck{yP)Wgm0K zeESF}=Fy8byr=lM)`?tQ5fxvcJye>ubMic+gD z*fBh%=NiLPM*pvKjo~R{c*@o_hNpOkI?BrI&~=xk_G z1)LI8{$c@VVR&$I_*xaCdN-#SFy*L}CEYsH|I3R85uc*Tf5L`L+X!Sou@hYlO}*Mq9j<#F8+O(W(@u4wW$Vl)da zUwzT`PD>GM_qmoH_mq_0xs-KV_xfNFTjvf~rQ6z(Bo~Fq6Q(3NyS|!|+ZC5B4fzXmLyl1r?lne9 zbai%&k{H?37$pJMXU3bx8g0%sTG&2aHaA+>epwc_u|`{0zsDMFEsu>g+VJ{pYP5;& zyqYZ;3+6s8R?+F4GF9+MB3Zp1;F)5}jtuLu4BJ?SZ7jn!mSG#qunnQtScc8H#xiW3 z!}#pWu-)K0phD=E^oR>KDHx4`Bs`OfW|6;s&Zk_RYYaOy>Rnf7NA+%GPosJV*Js9?#)>;*#T~rHU^f3X2DAOLg4sUjiaWbj#>ixFL5w#u zZyk&`v+W`gEgd~j`HQ+USoK004?g&hC=@y{opRF?TWO1kCA459X9X8Ak02u8A0;a| z&aNGgGhFYaY9?*z!3wFHm@^bSkz%m{m!UBxRGu&uofJ%pNKLxTFIzJw`Ny@V;Xru>Us5>}uco-aJy`poIqeRo=T*Ieq0{HCayUuZ4 zEi+MIJg0t+6f6=&+$FR6vb9(WZon)4i&`fI~igZfeW^i+M^Ii98@9dbIot>Q>lh=RM3c;J1m-a4=JDaQn zXTDznt`VrCZ)V4y(2^I^FaE<5S>lIH>)tjKa@Kw4?5z9d>%Vs1db1-4o4>+Umo(Z@ zjI-!Ps2IOIf8i73m*+1&pcr>L(^tF-W#!rT?bD&VZDV~TR_e$Leyluzn#J=SpS7{O;%!;u)!!?1I}(Mx>TmizaJ(W+%l!Ud`8L0YwE@ ztl)G?&h9Bs!2W1jv$yPb7oyC%Ct}44R&NEp_^0$iPH9U2DY9uK(v!3AXP4(MEWJtjm+HcWnis;SGH88)&aRHrjwJxB+&(kW9FclnPA(&utu7&$-#l4-S7S z?nn-U2z_Y{@~C2UAB-#2>lh)hL50!_i&3azWivTqTPw0w&bsMcCOsgsANXutY{a#9 z_M9qXzj6*(P$v6dDhj?}%h0N-w?b6L*CtUhsAE-HX3SgOq zAM9k~D5iVNC6P)LES|ywl2!CmD#}c;81<3PR_oO@tzto?=G4v&WZ$y0SjLdklF^;s3tXReUpX?`@{eK=5N1w`r~hY_}`Pi{pQ)9ew;r2yA;{a>ooa!P1EG( z|1*30=I8%;{p#ocIeY!8Q!o8M?!`~^{E!H9e27G*Sf;Gr13T~YJ4N^%GBBvaiRWb$ zEOfp(zx2tRZ_Y12pvZaR^u}FNZN8Xq&aZXvT4!TtUkph*Hpx!Dn8S*>W#pG~kWa6N z+d5~st*TA+jS$O(k@G~9vFjIVmycHALV;gdIOZ5~^v&D1-@Ll~>FoOY)z$gg?bX{i zKb_xPUR=Jpy*hi%C&ex@>*lOL%$6TPnV&r>ZX|Jdr7576OMcXg&VU%;cf0?2HgDf%9T7MHd~E=^bF zo`a+@9GX`!(xR#vMeW0(^Q$EUYS+QE zXt0E9qnck`bgOrLMc*E?vdL&54x+H4%NgQih%goe;Z$@-N6V-Q@DD4r+XoKrUhqplupXro;V zVgD`R*Tv0-ZoQ|wd^k0IB{EUG((I{EOaF8IB#Bl+NV5--lO9Qu;5}FnNy4Sp!dp5D zDym71!B@7a>@#OUk8gR(q@rma)a&|bP?-vm$jTV&Fol9?*4>)%X`?+&D>iYPXO#gk zriu<6h*Ejz5ece(dCo{s&FI4&lHBe%y{R>>@D4q$T~4?3uK{X!@Um4SD4}ntW0u*N zW#*ybm}Lgn2uk>oKndr%&sx%iSsT8#kHZEp(pE8K#f{h9(kh19ct?xuMlTW1mRoVv zrXkt2sYAGYPNjSh#Sm0D`-&eGuE)yeQpYyfCd5YI58s2M6x)w}qZI4>)+0%=QnF%L z=+ry=%E9=JZE)l_<{J6U$Zy(4^x5;9-lj*G<&O)qJi@e$FfAiY%Lvml!n6#b*9g<% zTq8`&G1@X^Oy>qWoz3b}isNRk2 zX;kmv`pkIKh!8U(#NahXA^EQ{3h9>=h4gtN#1K*jIxfGxd4(}Z=xcL&2)GRkrlM6h zU`If9=wV%w{$$d!Xmcm-m@j(ASbkB6JZJGOQlzi!+BGXCCPz2nDOW_8TrmU)sgYzz zjO~(n+v={9=rNiMoSt*d=yOOvOIa$4^^}~Fl+g@oOu^@jr@WhLZQkTBTfNDcIRUBo zzyG3xX6-`25TE8!c&{^;0(2vSplg{guNce{Ofp0uXgV7ql*G*kE;IN95*m zc1zB#uMSSX8Avs5V>tzI)lW2!n``i({K*3M6A-h!MfXvamZsOm?01fVzk<#+!+Jq20% zE*J4TsNDM7@=FVn!6wt*@a>m9UE|2yFk!=}4Sp9LuNnTW$xhch4*R1Az+gNQ>& zr!NqoDu;UN z@9XQ*rMaKvBv85RRgDgm=x!`=A|nh$j*$Z<(y)6J=QPU%7$Ec{QOtRlT17;VElJ*0 zIgjs-HCKEmbeCVe8V-ozoMiP8y4*X-utA<29oXi^Y+_#4~ z3aU|1d1yEaD!9fx_pWQ?HzU7kzeax3`K^zW=Z>&1f4;rFei5YhIaz=BD&+ON@LK_W zWJD9}4MUKMFwBl~o9$IC2}mxo`In|m{PB11-*?iquwUK1`;~Pftml;lvwF`(7r9yx zk#eP2d>mqHv~SAVn- N=_c!Y2C@W4h72+snEMZ#bk0~uimWTQLc;2F5epcI5` zTF&$^Yu9VQ$|6t>JSOZt*ZavN@>w?-lGz~d)4Ydj2QjCPPo4lfGI5{`J6td@oGDj*cX`x;Th( zKISEic?o0S@0gcB#=L~>Ys^dV4s~Qto7b3^;9j2@ZwgC01qEzi;J;N4@ini4*r%*{ zbwUHYJsNF&4~|CL{pdFuZ9Bj9NKPUfC3sB>>NhzJtA0tsr+tH+HGA}Kv)~q%AhU>e z{spZBp?Pjf>46>x;iuo;yz=s~AcD(C;Mv;CbervMndB7`~ zCly6^wPudRZO7uaV{zNDxb0Znb_l)3;~9}5Lu?U z7QY)##%>j$O62Jx5X;K6TS0mw;;zOgkT%qLbJm$LiTArRD)Iic=+??|&%G+Ur}GKc z`-E*{miT(u$ldCE>MC0nOm=^Zd++xyC~360uy~V7_p8+_n^V>Cde7OUnY#Linf*YG zsqZY_>g(rSNuqd#X_=tD9OTplsm}*#$+x}n+TP?`Vn$)5|01d@&m7YEl~~^v#>=~r z?IWJ;Y#;G&YWtLbYul%~H@JPGdz;%Q`Zl|Lu5Zh4w&&aUTWtnB>t_P?DL*T~D`+8x zXsde7?W3;tn;IGlTEe9QE$xo2JGZsBSKG~D7gCzHzOirJ^4)S*#){>=LmERzM&Il& z%QriQj=0wtI?~nIF?3{PPh;o^T%Q?l8Y`BM>Ydy7k$?Vz_~)qJx!0)Pb#-=B??(1C zs&{aGX1r;vSUy%P$7>8t_FrRY@-Hhi`E#yV?l0`!;fmU^vN!mcU-6W;@ZJV>+6sEt z#k`>z*{sn`BfX1TmmC!IA(Ezf!WvamdFBSf+~by%QXziJ}2jK^6=pE!|HrV1Y2 zR0Zq%YQ4X9pSQ){G3g*dvknYYk~v$T*jzH%u5X1^aDUSHmaX0%uQQtRh!pIh;I=|J zl+n=7wlXlu6f#6)hR1{^tkD0*`HrieYdRM!6^aqvII}m}+2@a0n|`o=A$Wr!U`oKZ zMb!1yfdBv5`}gk1ZDU^;&foqNSaQ}YDN(mHc5Eke_St^5XU4Xc=b|IY&XdRX8KFsZ zHzGE`4S=ob9G~z0EEE9VngqaBYdjMZSu55^WD(b@Le;OT5`TJRn*&pjJnK&Y++2(E z#5wbGBJ`$1Gpyy0%`7d*g1kd3tdb>V3onT9_J=$tBEgc_+}$tvefe+rigXfXKxXhS zr)N`i`&{t+#eWw#v!uT#r2O};r2q0nlWyh?y)rY`pFGr$2J}z-1YHIGen&_*Rzv^s zd*2^Z!TbOC-1o;Nmdlj~FZdmn%U7Pi!=CQLnCJ&y>IYtV9smm70RRc_D1e4{`hbRa z4l}o3`)LH9DBKz7ed34U6M7g#Z*K2ac;S6QPp6+od8E9$p+|jO@wCW_I9%nyEAudf zu;IJ%My5AkdF_N8QyzA&8whFxst38Ng%nM|HUKUk$WUxp8kOkQZrv!PphvjE-daFmytUy-@ksxx z>IK`2X8*Bca&{T|twGXB9FijHF6djcV+Lqe{0f9of^$`feq+i)@FF*rlJFYSOmB|e zGdbBjSUj%m%C&o14-YoANuD($>YfnDQI%jt7$nxZh7AvVDUeam*S{K(I3Qo^s7Yt# zF-OZ}+k!-9y5pI*cSuvC0?c1}Lr~cqVIfP4n2U={t2yc=sztor*yZkO5!v1;`v7Ra zbXPEw3C$?=HH{u-!(6Cfw|j2iE4)Emb10XV?EL!K#RbZUQbfoTm!RwfVTtZZmg)Z% zEG0rFT=?LJRm80@Fu%}M2xRy&!vw*3)dMP0Os4U31)y zw9<>;h_pgSWV)E5Q%kKQe9JQzy(tbKlHAap){fEan%1zDzOa+>fP{wq;aVM(y#ZXF zx#c-wQsLw-wqD$uoUrT03mxU+%7@-#wYaU08FNE_tvFb&&XZYn|21D@q@@yOiB?Va zTQ9Dh{D+l`%d#87FpEqP{P&t|-$mF|YhiaaEF{jdYV5r%5KLrQrn_lNQWrr*S`+Qfu zuCpsXH^O%LzwQnpF1r8p+E)kA_@_XEIj?|L+KqyTS-rcpN>WJi{E_PFF%TFd9YhK$|X9iu*8StO5VF0IcmOQ+{cgMI*xK1$GD8c z8E^Vs#gCG5!CnVwMpaz4?(^CUZZo<_OJJ5Jxdl>uw}AG`VwUx~vnMF`|M^_Y;@-bv z?_V#NzSnvzmel`1a0eR&`{H@jJU_P&KkQ@C0{dK{71wGq0~TYun$|7`zwa&vpOh6g}fi7FqvXz_>TO5T(j>2$nk~c!qM!jLD{~rVLNo0>++#$vFr` zJ-_<8!&69^6%})3yyC*B`uwRMod{8X9$tg+6Xp^JIV$nRASP_2;s z2~taH3t#K2Kfz!!bG$uuwTwjxV(BF+imdlC6L#EL^F*XhaOs}jncW)#gRb%Jqp2&Y z?kLpnzj&MV=!Z_?Mt9soT3%K5+kV60GiF>Z`&sBeT0*H+Z8ql_ zO|HZRoH9O`q5cT92dvG4H3%lf=%TM5e?>4!X z@v_&iIzh|OBf;5wXwDeG6oR`khsMz2+IPp$;U|&i=kJJrj_O^#jp|)rXGisJWKW}d2kj%{O=ISmF>{P* zW9Art8#Bj@nPaxwhnYDB0q%^(;hKJsMC|TUcL2BOPca+cw~b?OJI8yc$l|MZ!BO?n zEb&$4h^FX*)e(CRrf26CK^BsPuV$3_A@O`(!2l~j;RR5rfeW!9h!K+Z9x?e>yDmmMJ1Vz7{})iS1*0FV7=2Ygtf$!(Tts>eU{D) zDJLGKWLIUxhoOyELV~<0nlF7Y%En%mjNtr)l!_gdiLkN|QWE^EtXnAvh{Rz^z=O(G zy7H7=og7@SvRM$YgJdYebYGk+SeWu5B`I5_C%c9@xEN(;(a8?Vab+l@3p4umc7KV< zhZUDZKX-;q@6h|`f|4HQBBFDj9k$Y*xxp^^y#eK!bYYd=eAtP9w}~`tf;0fiQnr}Z z2mK}|CNT&u6klOQ6F9p#O=Z2m9mGwxxjDH2k=APlI?k+x9~X2%wOV1Jy8q;} z?*2LzxJ*WX))d1iZ{NOEtNd*hvi$a+eD+p9e``i*gi@E<^r+d=wnfaCYi`UnH|Clf zbImm#7<0`vo)~k@jk)H=TytZtxiQyVzqMz~H8&*SX&jjl-S&}ZhZU%3$A#=GyAfMGt!%=GP_;OI|gNv z>usgd14GPgS8P;~9O-*E_klNik4L|odv7FwBuo7%>Nbzx+|6~&4;<1n@) zA$^-7S=M#YHJFv=K9@)qdCrA0rq$a1r3`VAn!}u1S4ZbH^$4^B$cmyk$_SRKN-=S4 zxN?VqwKS)8nudlMSpU{(qKvaV9bPvULrWBcFxS$)WCy?p8WYsXsWqo?X2mM`=5;y6 zR-eLxJVk#(w{LR7Y#PrS@=l#i&!$gqI|*+;{A7AIJ-gL^sQLthX#LI+{>yEb`=`%# zF+nY=ozHajKjuN`d*G{OzkB6fk2v({D-CBxMy0ae3`yE@)10Rpx+1&?O2_)Mj#Je{ zAxSt(;o#n5s!+ic?YhmnL+oS?QglTm;zftt_~1C2!ipwwNl35j&vxV=HMGh^p8(## zUHpu%R-9P{Tz2`a<2XRQKeg`E*o^H_!*<)P>g_khsM!y@Cl}4USHM^dfoTyph3$Kj zL|96?U__NJbcK>01oO=@~+r#MqOuK+Fe`br3L&J9kOYLT}W>XLbf5N zENWJNyS8mTSv?(td%TO`j&~A!n~b=V+*D6Cn25cKw0Lmub;PaPqhCinIT-@k7_4iY zGek_9--ejjmJQz1+?RUcF_JEX(=CJn&k-?rJ4N4x0hV!K>>w2TA+dCku-`Cf`olyq znf3^P>BxDD14|E%ev63M7!q3&hm;QEr2Q!AD2((j2P#-FH+B0X0n5QsD+CA#aGtdwx3UeVp2kPfO zLC!Oj>x2Ts_Zt!(;S+SdW0);X6D@e!wr$(CZQJ%~+qP}nwr$(CyH9sdzu%p?|E7NI z9T}08l{+h+sM?V$S6pLfBC@x4=2H=(=pH|4?nluhhD1JfsoMH=;0f|U!6H0Li~U>^ zIPDP7kI>y?R)oj9x{D%Ne&v5S9cRZf#+v9+*(yEZq=Y#LpEylBpJ8v){${qi;8f1a zI+VtI8p?}^_RY?DMj5=?D9E0y>((QKr+kyR)vBvsC>0MMz$Phg>w_}%eGJXkY8Vn6 zUlcC7^F`7!uph=y(H=Vot!>yY&vf}cUY61H(tdm_zCB;of0Nhedbzv|ucP_8J*3k0 zeq4kPW*y{12)!eHIVeg%TkkUHM~K{Q)Qq z)&>6B>OCfnX?4nUxn}lA<(@ho&gh-79`b7+Ka!6MCQG+d2)GPmq%b8N=~k&X=v36pe_lo~2)8v;Ul4Y^c>#9tw{6EAG5hMzQ&_gH zxPcR=T2FKzUsg7fC3?Uzc@|mE_?{YCJ$1BwP`~1!`bhJ75Nd+YU?t7{=S`9u6l&-` z)Bm^uQ>0Qahs3yU!O!zDgO^O+pCC=1`0-GEwgG(p&+ASvqx0J=aB?+K02|V7hHUZxz9jp*+TutUG6#p?J!Nb>)P8r4B>H*eno2s?h z+I*9+?!g2k2#?}(j^xtI%Ldt{u+DSsU-UrDt5z)Wl^{jXJZrB*?N#({5`>4CkI}}M z-EYj5r64wEHwZ;cUf%8v$6$AE%yJm)Ln5&O(X!kMQrrS@dhI^rVU>$oKGjJ8u}b^{ znR%>5VPxfFhi6&AChEHb!-k~&D<>7;m1j(IG|e?gMbv%Q#7j{*nO-fvFDkY4@26IGC@y)6VpiSxHzMMcQeD9TpCj;1D9fie0XF@`Vf>-8YCJG-hlm zk$5F#A`jX-&_XnCW!O2MSbi7K@NAqRP8wv^DWM~ZZ9fShS*|WQQ1o>*YRwdr$tvN+ zc$C!!KX>km;E+AJ)Wh`YM~0D;Y@Bp_?7Z|N_Z?B{Aa!zwFN!;s#Rbt%$auYmPykMOXOk;0Qshb_wo z=c-kFT1gLD&qp_{iArzhD@KH)Yum$a@e5Z1BQCCaT34Owkyd&8YA@0IX5aTHfDxfQWtLkCNe{B{rHY^<8E=5AQ`>qVXP! z`tp}cAxPV(u%=I(!w9(^Jai=$0t0c#?>b-~{*_CCMKtM&T?Tg{#YZqv%)~a|H8;Y9 zH&2pI@Hu+}=al%k@ zS6G?4I97XC&>XtwU^!Mdt>_AB_|gs<&&~9lMP52oLl0pBw^jD~LkVUGlFkglA7;7_ z#0K0b>+qH2J-J#{Y;>l?VBttQ+x z@TABUirdfp!c)mpE3AZcDiwr{{J4Ml8%F^>=4dJU3PMoy!rVRM47}hXhTihPv*02| zKk~BggUJas(kx=*8K;_Fhb3S9(S$nk$!m16S( zmxs2@duVay8^9AuZ+xDCC;I%lDf;uBUuc_pNTfmU5a6ec)ga|e-kZrT;QF#>lZo)3 zGR2x7QDlT~7@IxFpyYH6=(o4f%2ZumRF?g?Pf}jve|IZg_M2nGht!03r473IzS76^ z)FYEzN#KzdN0h_p!1;Q7VQx_u`_=Bp7so4{KlaaxwHsOFMg-8*%zhKi*mu(*tuF?i zWjh!Ufg0K4q<4O5jB*ng&zjfVTr97+mj+^-ZKc zV!@qSh%pWx;%I}ZzA65oxTNC!J~*72fF3pn;+4~i)r-zP6`v_oRAvE|viAaL$*Bkm&wk%S$z>?)DXCteG?lC!ak{-L zRo^!udtd+i4~sgWg z*qBLW^nVrFw?)T}!P}p>7kYQ%{_rv~!K_tJK{;PVG;;L!**)W|^Eusxbp@Z(1M-qd zr2kRG%Zw5aXYHyR`Ey`Rbh2r;3mQeBFI#TSf#Bc$;6}=~4?6VzV>#$z~em$QQmZ7ENPX z^*Jyh<4V-v&ek_&=j-e_zWDuUO+`61meA02LrL$RjheKWxvK{@SHD%toDUR$w z9usqAW(8`&{H>cp35k}}l+jWorKKlH!ssxHGl`9^R$)*QUHW6ARm*5^A4ders*PpeZSIo_%I{NGu;j4RD^r_XgjL6fn z72!+A4iCdFLde%hk5F?-w?e8a1$O1_3cHCd6~5VBhk2uEw}2sSSHwnayVOl*WvQzM z?Lf7w3I*!IdV(dCwXJ>pTk0?h^0KTh2GKxSeZH*DwpGF4`M*WTYgWpt=a=~V687L^ z@aD#e4CM>ljKwJH%Nhcm>9b}}HBE{{VhZ5)p2GK7eZFYiU8hXIt&D}s3Tpf6yb{dT z!ke*^{KcB~2QZdPS7Rpn8)q#Fx7%0RTc3UYG`oOTNJ zwzBjxVlwc_kCC`i+s2P-)=Dbbk#Ieq@y|j+F!fmnlA&3sPeU_e^`j|)!UBc!l=wzY zyz5PZWe!)2U#RakLINM#u(mBU5wy**6d@@pXwp223N3Sa2rom>1R+BsZM*`M2(Zf` z>n+Z>zkk(OMC3Gbc0Ni!^ctAg%!9vdM0eJHrR>Ze*zso_=iGB@2Uq8mDvJ_o{#M$b zEOVK1rK(JJ2_$;7lh9nAiL&N@EDWz)v3!B6R)?Wqy0U_!eO8MP0@4Gq8If0>sL-Z2 zMCl<7+`BAcjWuO~E7vpJs+~|6`6`GBG#UZ6dTU`%ve$jO7}6}1T3V8Fkv;l0JU6{J z1}jeEzXj>V|Sa&67CrIGwjEG1J*5Cez@b| zd$j^C!vfwSTj31uW<#aZ@lM*W@>p_Fk42Hq$EISywzRgel}DM0K)*}%qsY10o8fM? za6Yyy*@3COA9nkZz^eI<uj%eL+TIL2?jSl7y zSo5UL0(#vP6qUxXTEC!f<{M_|~!yO=&tlkD1xp&T-bUGnV+^ds>S) zE{v&F2K5~e9vLlv#O%Y^A54FA>X4o?sJ?R1Zp&&VDI2Wbvyk31sN6pK6OB4hd013| z&uFHhQvr|1gRDCYbb}Bdl&uOVbj&rbPx6|aB&2rQ2QbW#>a5 zeEaj?fh@yG5TIw(+h`c`+Pu9(o_d`A;7#WkAbsg28Brb~XA?=R6JBoqJhN2|cFJlm z79*aks2@s4!c|`}2{Xcz@(90sNonTfeUhXOdE{0byzRwU!^&ck72PJ#&G&fBR_58ZIs9+i6dkOB!yqA%*IZm`;tk!9hF#xPD$$jbp(cN!C#h^q%a}&6- z`@XwxF%Az-C=D0i{~@&u$4D(03yWKX(KYdW8(IH7)bzXzzCVlKZ`aLy{eO7h4=NjL zd;a}{lly)+R?FpacYHazn&ivkdB3Qv&G`qAeZ9ZguPPoqj5jkAmM3xJE=$Rs>-p_t zs`{wO9;-8Yupr*sRWe9XDotF9KZS7GW>_4S7Kiojl#t`u)@3dq0Y4c%tx76(3ep z8#+36uff1LL(gpsD)?vU7QU|qXy5ktY@Y&PeBA2+Adu+TO`ZXG77+& zY)WTo@LHJ~AVd8M!aTgd{vgYEKfB@Aj4v(wB)EjF-(HcVLMz^27Qjh_@nuMK0?KY% z$oAX4n0$ZzO^sWGikM^;+^5la8FEe2F3(c=1I*ZknV^|Y$w-Y^>uso%llc!o|8uor za0W#_ow-Rs{NpalY&ar$cdem*W3`!^aa(M84F03^=nAp&Sc!QViK|&|vo}NJz5c$F zhGO%6S`xTKSTT_iLa}2!XSU_ry2o?*-*HRQqmKrBmaW&WiaLs;W=T=|#SNe=Th6WB zq7*w56tVBe$Jr0xGF|@9JKy|1%5ON_Ty$aZQ15E6TCN@JgN*ar>C{%8M~~_U_nBc7 zr6e4`58hUDj7aM5JaH-~kh@j>w=U2o#AIMx-C=3l)JI7F&?JbV_A_msd&PY0QYWnKR3BXvRl4;l;hrwwo8lR8P_S7xli>s%Pgk`A0fVpHPoV1-(@X>EVbbnYjkT_@rMW#AJDV9||q6u+r8GTvk&8rwP*4hl-czZB%_2`_&`oH{YXv!St$#=`YbT&4!0`T)1 z`E7CkKT$ks!wQv;@=dFUI-%R`4mLmfyDC3ah!tqfMr0MF>pzQXbTxJrHZ^F4JfA13 z%In7!By&ij>)11p$;%>@(lB}i4VxnmhAjn>9f+ui4MS=w9Ff|1Gk+s!(LF-ch&@9{ z+@C<^m}tm63=dQCYNDI)b~7(#YKd7 zwP_+T&it3NyIE}-qh@8*)O&j`kuMZv>7~d*zF11@caL<+VNMIE9K09rcHP#F$y-HC ztcu`KBL~=VI;<+1OdBF>(>AnSG9>Dj0?hJYfn~i)+wmI`({|-i4 zs)B4M!kT`C{%UZFckKUN(}x+H5J4d}S%~mB`*&)F2rPT9=f_l2YBG>siz?#&CT);1KC}^AB3aT%ahMoI=IWFY zf4D7r;sh%F(NJ;|COYcWrs2wtfraG8Q2Wq|A>=ZnB!1IMYyl9r6~t5yT#}um|6^** zO>)|evwkt<1XN0ds7oCxe+!*N2_fur+%<}b`!bobmwdYC-a*UfRj=Rm?>W+ynBreC z9_Jr1{s?taLtf(xsC2$FTd7_s<=KelEy=2~CIjkY?)VI+-ie7-h<2B1z>eHJjeK|& zQB!J;RA7Tf_`6}2;}O0YjC(aAJ&Wg|hh)vvsGv%4MT@w)l^i1$mUokb-FgA%LuSyvY9V&%^tTB?pvFl z;Dt}J0H#~Wu6>h^VvZczvv3onI@CLyUB<04xk`68v7Vk; zhanmP3yZXOAtJ(5hM@5qY-IL*qxFW>R>gkUE^;(hxc^QZxqh=Q=pp6Z`G6p00>!w5esmd&_qeNE#J z!2q0^faKK%huHcM7n?0{O$$wgwbJH{SHv=uZqHV3@8{c7US4cg2akiJuu3`m(69Q<#`Gva;Ieh)J%kmm)2nrChf$! zDY(yCU@q}@<*IScw%TBv?}ysb)W|Y-q=>T8I$>vYO8T>Eli?Bv52ZOn0r;ZTPV40q65rx7#I3yd>IySWA(Xm(sJY+{$-2kf%4;v1&$)i^Mvh;#VwAEMMVB!&9jFayvssnrM$`B z{;+tpR1nl<)B25ZQY&b`SF-=wror?GSzE=h#<^Jlvz^-vQ_tGcDu5eMTcGwBbHeb{ z#T`RW^lEi5&KG7)w5x|N*6mXc>(2Qomv`mvVYW}#Vyx>&%*(A$Dc%=mA*^zX0@y>7 z0$2q0?;x+U{=XyNL&gVm&!nFY2W|D{2Wae+bAt?=__%}wWE8as z0Zigt8YlvLQVXGB7d=E} zwDyRe&G3v`OJNehWp@;8-BURO@D3(up??c2kMaV$P`4tEtBm zKjD*kgMZL@cKLi^_-L`b7*N{qUeU#TQm881x-FN6Ss~u;_Duw)*r{g&#?7(MvfH-B zTpia_G3z(1W2v6qDgZMp{N;@NTMWH$`axWHScx#O)wS9*ssUT2+8W{<#;L2wjf+(J zMqEQst)ZL?hjckx?e+>%V%R)s{c0|7I#Q{0I%FH&W$Lw75J?9}c1AuCzK@rdj9OOg zpDC}dKcYa*men8kYp!#OAvR(l=c%Csww4<18zX+^NF{#Fuk({)G zN9hOloXvvdBnkFsjcz@}6=IxKIdTtbnGr!v$~ldmAr`H&$~9Dx1bS|uLfgh(v?|jH z0jB^J0&q=z6md(VPmJ0o805>rrKBE8qxz_fiO`1&*5G&@KF&^4Z59p}y5mqR}0w(H%~GIHL`%BhjAiZC>LZ90g?0&R}e^5y}nf1?*O3l5hBd0!z?kMp>v| zQfc0pAB|hneoKyOtKeFl`O7f3-`q2wa~>5ksM?AHXqI?>W`{+PLE$PZ#wjg8qm&gI zSNnPhQ^cJ1ZxKQo3LI&w;lN=1Jh9)iKRmQL3e$oIg-n)zMWbiR?6I;H=0wew&Z020 z6$+Uqn&&ZCI15G1lpBAIUp(LRX3Eaf7{7VwciCw)HU+p<)YuB`vzFtpaN4SfuZBuU#Z+>%^Ho=V7 zsoaaw?(H&06hTLoxrkvO{@#yy zbh_l02z=nm@BoG{lNswRVPb{p@#y5$94e>X!1}@BTFu%w3VZ24lA7yJf?4esde%yy zoNGjAR;*wSR$`nYbbCq5oMk^1tH=?#!whc#Ts*2Anb2vC<_7gbq@ADLW(`*PuY3){ zF7taY44K&$Myy<$8S7V+U|RUq*`bEXceJe4I3J#_{X ztShe9ZMmt>)>3vxJz7Ym;TeuG#o&Aiqdvuh6+;delP#79#oNU#S;7@ix&y_O2>GP{ z#AHssN>eR^0{$c_SAb98NwyWEyX8c)6vIVqdPJpFkA|;qkf>gEmk4a7N?H4HQSYGE zc}C>HS3?ko-X)ktZ%m#VG@k2d@ii2g)#$<9w`Nq>IQQu!W1rWe}l5fN-n3 zO~lD0d~Q!03{XQ9JdspqBl6vTD5qoZ7EYuN#-*4vT zU*hjQ;E27a>J4>>Qs3ecNp>TWES!*tU!jMU6z|g9=%JqBeJ#z)TACMib>zs1$3~j> z*AVG#_N#agXLwi7iyj}f{(P0?WmO$`Ztw*>!wXr5R>Wv2GxGlOxZk!$;SVj%r`>Jf2Ch96JYZ{7o-U$_MiWWu)u3zvi!ei zB-{U=F4iN)oTl#@kYtHg$=PvW@8VkpIP!B~!-5^ zb~Os<{1xr6khVlxe+BJd0R$HhY0IXWp?^DVuckMe#esJ#0M>{lG`1#V@P7>)S{TV- zGp`I`w4eR2%(OC8zVUx7zjGpth@%5qG`0VX5&geO)3^IOhxYF0P3o5ar#!R!y?Q;V zZ1-=&BN^}c@xT378xS@PM+h}_4wICiAWWY{;0E?(|BOhwGzHaB0eN*6S+|dflg-lwI(D1C zoM0^h$Ch7fPn&(ml_kBms#KRO3mK17U`9vBX>K_0aK_yTXj}_kwZ^oDvpW|xWL;UE zYDo!9uU3^3!`G+A=|KezGkvg+QY6b!p>AZAyYNDednq0B6&jt0hE+q;m(i6vA&WaTt=s(q>d44yj!?U?u5 zEOS1)P-;MJm}Y`tK@wT;N^CvGq++=?FAvZ3Bw0yPZV7jMvtD+NC>}nk#L^2JZ;&AG+e9s$8|(E>ktiT9*HHz52Aqb zbTS8;7mLVQarKs+)?aIX?m3z!Gxz0eltH>GFYU zYMi43(hOn(mHWi1a1}*6U-#)jOvA8n?r1oI$^)|T_<8AFZVth2m;3h#1*e>n7G-su zv956oDfkAgz95b z(6Gx3S!J>N2lF8}5svAj4@=SJ_rWq^$QJ5;Q#Jh{6k!nu8$gW-V9ADcI@A}ID5A+> z??q54c(;K@YER&5nyFh#GRY#pY|Sqib*9A3-X%XGr9>Xhvo+dQPDZbTIZU>kE$8t* z>!)VcUHuouQTp$9xf``+NU_o{WuS#~YuDZYB{7C#op*}tO08Mss_ zNY`n(z1vFmMfE{b_%u5=kHx0+vdCRsnxH{zk93pEVb>Mbs*{+ojE|Qpm$}t2ocy!Y zq(51qQ?zbVyZL}3+pe!&oZxI(S}?4n znxS35@m!Kq@)y0Y(#XH|$iD`=gPy1~1FosGb*X<*uS6C+N8P!Cp1!0nuR0ij)C zw?Ek`N-O(cEGE3|dgIYMtH5OK&{IPNbi7~_ z%^-?;e`}DD8^z65sxJIcT)){ywb|zitQ{^kDFYhY_TL;C34bGeh5`awOqWB0#GghB z@n&mxdU$=G3wD7kU3uJ7jZzYjQ5qyE3_D%j%3e1!AZJwblC-A~M6EuXbsJE)<>AL0 zStR5xCS(LD`Au}mC&RzQC8yWZ_W93|c}|0Iukn7siQtK85GS9vbl(rdnLGf2Zi=Cb zeM|zJ*j_f82-*!Gdp#{90P+6Szmx*`+Bx+*r%uf?7f$A_*oMGfD3_cCf3r$hL9I$5 z&rlOt5%!hZrd|U>H1rWphb4soC}$|z=Zl{|nrX|-H#%-h(?&6?LaC_lj4Kfxky5C- zDRNboiqd2fdo>SGXX|@%k{+{ieJy3=lC^n7G(hBxk&HLk;xL^thSM{LTKBZjIfQf!+WaU9={+HiA7JJWnt4(e7$@^*}fV>Mj7#A`HTOxGK`6f;$E3 z7IU2Uj%?j`8j}bON+`qT-5UTQ}{dEpWtj z$P`!pD4795wox&7yu2K|N3OD={dTGCfs2huCvy)Tmp*Bbz>AAME28o=z}!aHG;<+* zyPF~K*$VS>`5=}}9I?Jc;`IpZeh3Ieymz!_=Y>l2YRwgy6rh1%xKXGGm|V0nm!%&O zpFkEEFL-eXYTnkT zR=9JRXy|ZmKQM0xZrl;J(~KRDvkF}yyse(TuXM%(qp?;^*uZ-??IO^A;U8JvkL;Yy zyA;yu=i>gpuh-83-jBQ6+oa7DZtV^jtzFdw4B_5E8eIhzWO;>kT)VGX@MoVo}o0&W=9ws(D=_< z{QGw6E(W~xYJL;P6eV3!&r)X6)615KET4O-6NH*x>R2lURt~l>!bH8Ia9!!yFE{SL zQ$4a#Jv^MRzpU?@biTYF?vQ-{&fhK%^|txGy}Z2sJ}ti1_H?&<-9zzwKa;6e-AwiR z@O*@Ky^$o0-QTbJMqTIzGAWjjYD-To@rg(Z1CuI4JElH59vce{V@p}QfNOb&kvrmi z*b~S5XI@{9*!EkkMB#remz~=py;TJ#A^MlVmyd;zYUX6W>+o zyIg*7&3~j8O2P<%2+BSbhA%86vi1tH%4NFSL1if?!Xi$9gaR?i!?zt9qrD6&siO8? zkZNAqoS=WKmXsj1l%-_zI_n;A{SvlP5Iqw^>bN;^1f?HBv$ioeN5;NM{7t-Z?o>ZFqeAm>34C^nUPP-|H$8QM*ZC6?v)4U_-! z@C1&>BC`bx+i+vx>f^;bFm^6*Fl5m!5{>ND5z(U^%krobHHRXer1S)ofl&I0fv&sO zj?1cBTK0jQ3Xn6bK4w@ebWeUBl>FgWBu)KC1!MF-RuaNSk5A}i5B+nB$ zoU544fLB_32yC@NNwbGk{%}2(K|vBW87|k(ih$u4#L!6HpC21~tMc+FJ}{xwp~WXE z&pSu;L2Z)oE?|t2nF`y}5Eb@yW%3Z4b4oDtGID~cN7kXLdV=U6t=P3f-TIs>oFts2 zbr((4EfjZZrqz#rAe@wkt(BR_0=+uHZlKS~TQ!y$g6K%=J-Jvg_x`$Shpk}%hR-f6 zC56PM*oqqCoOjY9+mb?io8n2%zPJu#TP@|6a1wdE%}&KHbV{&JsHuX$N%YN01J7ra zP;92NNft*VJndGK50NGiRT^*DC@`^`!jXh+!yFQC%93)frqi5HMuVf2ieDwJoy%H*E2f32hsqo(|)8l4kPsw$g@1L7~>;6KjpIo8y z%}V+-F6gC}~#9(4B~BlAK79)_tW1RG%R;Ci3x zI(-<{WF#+)O6xj@FF-JHF06uXrz+jG)~~G3Pl6m{X%=`~3yN&!6`&-YLE+@&=HcF9 zyc15pLm)O{s2UH>Oh&>#r<`S_KN)-iRzS&&P;w$xpl1VOjX4IEh)L=*k*4~G8W1F6 zfNqBTTsWu)CkL-E&d?Fs_36P`dKO|;-Tbi@f)*C94ppwZ!CeTUvJU!&eh|z^;I6*T zj{S6iqk@WnB$7Nkk8wpjxCoy@UTEdmLCmfsbqH;iCG#x;9Gkrvu-^Kb72=V{!p)m( z)l0~M-OVp zo>;ifKcn>5fkWhHFZkvidbU9ZW0f*o5bVPV%s{#aCr0@KxZbrQF+y z-H5j6fU2oU$2z2MtvTj#`kd#`{x#s}#9Xr@HhojacQvXf$~aY%sRE9A0l|18a}yiN zl-~nN_+uyJR5$iju7LJ45+k#FF@2Lr`=Q_>U)ct9A17-b*z8GRk^iiDu zgcV!KVJsA%ab-tuXwEer{P4CS>B3W+KOJhlITSiU$x3qSUJxozal(p zvAN}6W-W|_*^yI00U5)%i)Vr7-7%V;*Z1(!-$jK1Xq~KHUSH3zLniQ;2#sYhqsl)! z*uyv(D{^i!vBFfNn-o2U4ECD47NVNxHr=OR*ptenbpkKNt`kvlUU4h#)w|}M(n_gf zG#w4%CiHcp1ZNN#@f>Kt)8Nz!7CURDFO(7k8l2h?ovgR*n-0$AH2vydD4>{)?$GK; zYxAkT;XBq2Q&c{!>+{!EgXP4%C*!~QH)ou>w-Jr7I^A6?$j_49zD`O41i((gt+F4B z8{xdos;xdqJ;WSUK0-CLtY(T)GmUEW&OEvO}oR#U%9VQvO8NmlQ zP4p!Y9NiOjW<5C$?EZ@guijf|9@?|PXV>7J;^F!zt6=>o6UAFTE5!6xE{wH&QfS`z zpR84g^`D|*=YPfzzbeW9Q&186pZ>REG;f9Je`8h{dqMP~0D|WblOyH58XdXT0Sv?? zQzaKRi-o`!qFoKIc>AZ~FBI*}u!j$&o|4&693M8x&eZL|$&tm(^pA?@cZTz>I=~0I z5cD8a`tz&VLaa$|e96C2`a?M8eKAY#_><2L3trKi5b0N%MlW@xFxJgg=?8%d2GY+G z4qom;11IlIWV}Mscj@PQ(>HOe{XgYsW+P#SlPe-VjmW~Z?k7A((T~r+?@pAe9-Wfj zNoO5Tt&@tOz@|s5LH#l1;d4p+AEMGs2^(W0B)UQ*O@tEQ{vVn*-x!1+?@d<*>6>5S zcEy&jQL}tdGnV4n8e!Zyp+26@?^Pp03{KQZ4J@0_1OXw`M7OM-8%8P=H ze!7`iW3n8Owhy$&Cn12zP0@0ZXz*bu-LOQ`$n`G4Ni4G{U%-SmYL76^q z=#$N)61H?SewO2jiO^uziiMG-BFQ zVP6D9&l&E&M>}@uM2{oxc|<2C0eDCEzeUw=>uBpUYZ(-s+@SFeZ@z~OdWLbFoNmTT z-_XB@x<3)YGK4_i;9rmVyqc!^6nF>KefR(98C2!ckBGJD{pdd_kCwS9sCF5kb_uL> z_WST7`vn!aATZZ1+xi`ZK<+_Ej5n*DR{H-LM7^|t@OGEr8@;vvJ>(GJbl ztyXr_x}kBRvndEnU86(E?w;#-&YGWxRJKn}(;ZQL5sud=el=ti_Uy&R8he}@ev8>9 z;#CXECz{yuz+O!{pE%gtEpm~%WXsLs5bUjP{{xM!cryxV6wWgRZ4^EgS!EHyaFXuE zHI+zda{QZuu>F%kDjsIlT&5@W1WtLZzQK5Z3w*7#9a+K z@=ySF(%0yjbb#TVOWAHGmrx$6`(1S}Nc)C3cx%3_k2jf~l8uvy*bimCv;f*afIrVY z-UQ3(#~QdcJq_Az$QOTty#N%se5F&;c^rfdaX@N*<*jC!20%pwu>{^$m{Fo*PN{&y zW6V~0(0YjrFzHmA_-~5M9HN=Yj$71?;H&%NJ4FZSL>gT*_$<3tpI`kux_fG(@S+T3 zfJS`iXZi7R2-?82lP0EkrAIw8S2*?M!M0y=KfmSu$H?*R)1Z*0w*!Tm2@RMp?OR64 z$j3zryd;K4xM0m!0vi1cX%sbm8ET3|29HIL!IlAa@YN>)PugiT_yP+H3dl2Lr_A3W z_Jcnrl;|p%aR@rJP{E2mvoY1rp^`!KQRLb{m>Q%+2nKE`!RT!IP_~?CP}2~q!T9F> zC>T}EY3;9eK$MdiLm-Jmm4Ai=4+&Lpem5t}dv+3~LvEq)3ogFi!s#~sDUqWdmp%Ts z`(1#XWXZc}VF&SzdDK)T2@%$JT3fr@!~Ii>M)%pe1(ij8`EGzQ(5K|5Mtz@w9|bA; z(^>ajSC~*T`h}cr#8&1mfUqGDthWaf$+DNgfAT>C6Zv|HYzoOHnhT}_+ZQ{#+tmgt zZ%T`QvaXM~cY6o&=j8>l`S`BnEX!s}>V#Y&oRJa*Am5rMbcV#$o;CRAxpUP%1G3sM z1tEQzlXKoENuG1k5~)87ut=0{TaCM!>CXVTb&05>6iFQ+R7M^O0#tzGFQ<;?KFlvP zMC3+c`Dv_fKo#nC;E+JGyd;(R&Tkb_m)ex#*c^gyyD*xc=X2UQRrdFjz2#vrsZ9805H`NtZ z3yQWj4wUQ+`U}h>^c#qlLGPnE#m$`m%6k^s8JC{VcePZzH<*^Tzd$frFlvlTBuLfv zs_VK~JM?=l2W>9MO?wImYRwNsnAX6rNJGC`*LSqo!v;=_>DBwWI{AbcAR@Z4G#RA4 zBAQNN0$^Xi&QuE1e)z`MDflZVI=M6Lb=Kpu0hc+VNLF-J(e8*l8}q(~D@D~>;)jI4 zUhG3F&dDM$2&Ys|9I+~atua_D2mo+~6;=^Ry<{vg@xRUfK*h3E=H8f1i!3-^DXdXk zY+AFlOK`Jva?A{62lijoS5oDb17K{kv_L2eg`#*$1+@ftAHh$9l|@a|Vw#=)lr*l! zi&T9F9c&b693O)vjZ|AUXz;+Le`Cj-LJ_FWJ(Lq^L}cPn^b(iMEq-?hSj?2K&|b7X z=~no3eS}HjIh+&GdJh%Hwml)F@*XURY5yvJ=ImvNXuEqu|7Wsm{-4F$=D%8hH%0%e zaqDY3RQws{(_|lsklM5TKi#e+5$*OOFYoiO$@Tx7^S{>qze~7sAI4h-h=0&Ijx%Lg z7sN5DwGgK^)fQmy(A+`@99_!hZB=TrUG8bC;ZM1Yl%B(0nO50edE9xRi{I77Jkp8f zyR{#eM^*ZSXkPj~au*8A&NoXjTC;X}MI>~w3s5y27$V6R6w=V!zdzjO#PsW>ziOv8 z-(;MwkBmK0E)h8XTe2BOHny5P7grf1d(Nn&s-*293)1xte8a0eX~^y8g^io-DSV3{ z^|l_!`e0|?X6l)M62Y7rm`YM0DvRRqgB$H6$(lYNxTr5JDcCB)o-9(0z9!IbB_3S5 zHL~jX%I$xI$5-U|2GCY^xgO%^{=)RcP^!;l*g0mzJq`>c6Sx#_c1A?KbPL)Q6&h~a?R=P(WardD0QQB#dZ z2fvSdjqFRm4EipewzHRoAe@GX)lHeo-Ms>}=*GEM5&h~u^3K(YLxh6c3&2LXKVAeCdu3#&p?7*UjbX=6)+))DDy57J+e5akwaQNkvqOtxvDWUwjHBtbg! zjK1%JofOV>B#Ip~9dmIkf5Z~dv`e*tw-G8WA1~ssD|+(|PPGM(we<%|J0_VhfJfx# z^X!!Ghm+&`6?@td83k!35e4nqUk0bTY?rZt#4V}%tpD*X- zj)v*tg=1oSzYANn;ObIXNS>wCt1+j9a`#buz}QH-4nUMZq2T~|>>0D_z{)1<2x}Cs z`Wx0V9ddn$KbH>BTU{!;7jHY1aM(gKh2CK(5%d&_Z)G);%ODVYDXN`U>&&18x$783 zvL`Ewmr;i8X6;3}wfr+{9-t!#2Qkvw&55<|OOC<2h>;3a6I~$)%CCoZ#ZvN^P+bGlI;d;z2+75|l41T=fr7eNbStF~&S>H;`krY!3wezq-)S}DYn z%aYiFud8ZiG{ng=dvzn?xi{Ul2S8mzH5dwJDp+m4IeBjXA>3fmkg2k*mToiGe)A0P zh*zxx@BYwFQoE^`*XGie@pDyt&YZr-GU5mlyx^N@&r$zEV9ZSv=6SBH)oF@))KV~= zC{b$`V-Ey;@NYO|pNzwgBpiI6jQoEnd&l6~qNd+B_Kt1awr$(CZD+^WF?Q?~+qP}n zJ9cvNJnyMn_uM*PZq-~hM)&9*-{x9t%=znoUm2-sz;zjE<6ZlZ(Ni;}{--$zpgmD5=C>qa+1gErZMuhK~WQI;7CV z9V~l2$u zSz;>3Mv+E*yIj4dii-GN>o8-5rz44qc|7ewTAPWeR(;HY2 zF;OJ542UjT^>9`E^SnDf=YieFO>ni7czT6WJD7w*&Z@`+Iu-2=b)-BwmjVs2|9J_7 zzu0y;WX8CluO3W$36@p@26G>lCE{&$!96}qEp@PkGAvsa>nobuNgQ~`Jcx?9a@t{B zEi1o4RnL*;DVpP#4ujsVKg%==&OG$S*3(4z8}nCc;(N=R4Du99CsfE(l~$$RF~FJX zSx$O1w@l43G-yl%{_(9ZAnGsf!1ge5%EPDl*i78sr~proug_=vMa#*IfnKM0O&f9b zqItO;X|dl*Dax5TH$;i9e4jlyreErv&zYMUrhEZLs3$WPDd&hHGm~TAMt}Q?0a#yh zNoo0R+gUBBz@lv8wi3Ss8oX zpIG^bA*YBw8X9CcNWQE3GCAxE=&Cy+v{K_ za{_!)J?T_W!)vv^Osq|6Y|9}PX0bFSLcnpB8@LW)k%2{2e@bzQ0~jGof^%MCWZ87* zG3OsXUGiaOqoJ*8UGlgSo3+;{3|jmYo$esz{?_`;!Y^!7D`Z}wVHmV@9t`=p-KqKA zeB6oBdAZZ#cWXhxdDC*L3Hwxowc>{~@II^Av{aKOR*zb=R1Z?No?48~`Cm|JzAHmn zv$rnz0*XFx`nVsz-}EmMrI^O&2Ha(4XW|z)Xg747s=KM~twe9KdvIbKkA8YMPwplPvEkk z5i=icu|_>2c!0LbiG-nu(BJ#jpK-{E3*96BzIpFJ`Ct( zO8!M$)mxego0O1TOIik+s?%Qs2fsuA4vyR3>^+sipFW3k(N82dvH#s90xj?Cb8t)M zEo!I&&ZQ|?J2bh+ZKi(lZY<8s7s5KBYvhIu#=d~2j~J@fCW0mrIwrgtR|}vkudP5_t#G?0 zo=HtJ-B@SYwGE5&%X(N|TSsX55JG19t@TxFVXmcd_JHr;OwgdEA-ac3^DV^$*G&5$ zVAvL-F^A87p<~Z5#h7(vR2*dE`FrZ?380#kmW*~I>p0HHGWrPXiAH80ZfN!knayoE zMv`$_%%mp!p{$|Tpf)HY5hxcuz1X*DNvaC!k9g9hRS43MZI=?TCohSj_%dn2Q$r4w zZco0?UMjX^9^8ziIa4A8E*?7)+>8rLVr7CCR5X^TeN<9e<5N!4bl1WFR_n9d_Nn z4Q860%ELO-_KNcL8g4C3owPOzl`KXKlM$zd$OcSe%{zRjYo>La?A~`%% z(jX&nXwU@Th_zQR*qH2K0bioWvAhRV)|x-7 zjt&1%U^Kb3tQLFQgxb-~v0AAF#oH?7IC{OmULSCP@0RtBn}JhDJz%oFY5{UkkS}f{ zAosL`1#GLoW7Bn~LvXT@x(UIlRa5Fi_SA?ns^MZY_b;wp@rs|D-?lHFJrDbISnm#! zyoTyH;-zql53`Q5e?(kou@iX`{bBjmB?@1#qp|IHJ@O(LhhE9}EbghPavTVH|Kg7# zeYBiZKG&}j`%wM!E~(6`VwjO-h_s8QuDs^EO9;qqE(`INEdytzLQ67Hcp3UPy+?fM z$iCmmy;ys6G1@V_j_MAsP*gs&X9Mj)hrAOHlSV*K;Wx}|1}m2F;@-pqe0cl$DxmJ zT#N#ofA)NT?jq|dRn)`|MpH*|x5E3~1BhXZ_kvUs`B8W5!jOXvYX23hgyneF*m)+5 z#F9B~)AYFfRWlIZKyXC9YBWSQEs&k8AfgvZJWgEb^D2?pvu2Z8e-gqZ>IIlD^ zEn^a8?@03Wl(S4U4{{9H-4-BCzms8*n#S71QTf{#T1p^1AE`9tgIaWKN8?7JirFao zRPi-YOlEqblaEQx>s)ZoKHWSYT6OLHJyYmyjY&mqsIu2QD0vQ)^4fW6E~Yu@&v(2$ zx-m^NiPzVPGJ+<%-pc7S>o z^|jlk|7MLe|92?;&ZrxYi$9gf;`Ij9H*EdSCRIPVw~g@%U09WuuKyEn@!#YhJLbQe z;mvqD$x&TRIbr;v9t{4j{H?)CbL=vMKL>}@-43o^Rv*~pCn=gE$e#4o1S#l)poNnJ z+ja<+n4lSwkKxcw)MD!HncZ|B^;fXsLamw|0?R_h2&P;yNol`s zCJR4P*Zbcm^E*=mNviuOEw~)72C~-zf+`+kt~gc7lRK#bqI%qGUw?;JMDfi?4yKTq zw+Vh#KuIcJ9mr`)MAf&YnWqwd=pTsSj`=QIIk@ccb)`!(Xq>RbrtJ+PaoZCDHQ;Qh}5h7S;M=p!vL2N&a447;+RBu~y|gYd!vDNvC6!LT1gQbh)SZPlaKhyG*ZmVgCP)&hTLun2fZo?_Q-e^OqQF5HK)I3hILpQmK`A2oyrLU) zzH?>K?+$ul^?7;85WqI_qeb-P4dJQkWsHgZPnC{QgLC46j>ghd4Hv!J#^k26Mr{*4 zPMylZ{wx1+*uT1rbnKXdT)3dK94{D=yK(cEuyXUCg)3TvWnLQfLoTg_)qR&(j_ee& zLgaeFDdxYM`HNplUiY18jbidiI0`>2b~8aN&SMhYCo2t5QV1)TYrV$T z_A6r2F;0$~KL6VrR4oK7kKQL>Ox8spCm#a|j?cET_>HN&BjalSF>ex)iXPaLXxJr+ zcpI^nonom9P{FspPqrw34oJ=i1yx|qHrhxj3ALjaGD2B~yYu-k^|f=FfL z_VsQ}ySIg)@r=XhYX_-fYB8vaShPMu_<}#a=?!#|2`N$o0g{w5!gaAH)>F$htO5zW zzJ)OAgH4r>4a;L)>9lrxW3wm%6l3PC1Dc`3HOFWFoc(Fo32qX0&;~rZjcHG{t;zK0 z((eHK`IxcW)2T7Ijq&ekydg_gDC-4z34o+X+N3nCE*w#fU2Qqu5nT8cz`2CB(#H&ay>&b^P zGr`VX*w)+9ll)e+j}UQBmb>9J{W z@bl4_>d^nB5nIiU4lUA-8N;PGB>U@D+-U3kcRCH14myJZD0y%blzhxkqU9CWNeClZ ztxeuZJX8v+kyv`R2p|hxj*5)pXhlcPry6TgbA6SOct>o?ZCc}P-aoD|357aPw{w~? zG952P3`fxoOxTSiw#fsZPb>C+IUlOn2#nykuk6~n%a z;BE$$6mH<*c{Xc;Ej`^V>J^m_JLf0FBe*^0IqNEB+Y@BWqv1GyJ0;@iqnR$--v?WB z6negqXE&LrcAVq0(F0JqYUsR3lO+52m~sDaLA`A9&YAvMINhbgt+qxHjmHU5GyzQ- zFg(NZg0n!hjL3GSJ1AJI+xscx^cxzFoK!24-G}pU5T2K~C6Ltaelo)O`)rc)rikP2wx& zc<3aZ+^I&hyJ4XKIDuf97_&b@twym9sp(FUtQ%;EzoUZE(!7L!u*Ns|#W{=L50!$J zF{Gl>wCq16YfvO0G*v+`RrnV-V{sNdqw`*sD)8zg%hRZ)7^vq5Pp4RDN4c9;sk_Jm2BwhhTgp!Wp@Z&To8cfmw^{eBg{Wz=|S& zifPieEJfsQ|2Zb{|~4lZ&IlM8B9M z!lva&1RU-z2bSYNCOW z_V-;c#6H}-hw1MtIf5^qdM2SQt$25}rV%E2eD@2OcML#jZhwGq>ITQmj^L6^)*rxs zXw;`hS)}Q+cu~pmHRwYfej#wVo+n6G(Psk^go@FLXnxGCh@f<0=+L znFPG!xGLle&si(vf22IsO7+wzY^<=$c*#w;` zfGdfZiGyce%?2WxjgVcnBKzn0*~0vW%b@4x)J2@YRHUb~n|pVMfqG|obI z@l9eG65D|G0yrgZW8?ykcndY%@!r@M_3)Q-8MfGZi(B;LLoTY`;)2?Dh>A zG17K@oKWWd6oDoWvTc(;7Y@%D^P?3HiqYpBDTC61k8VJsC_9jkz`v&X^yta{-m(Qb z6_*p%YeXRIevnK%K~+oet;TV)mS0ww{072>Tmxx%r!3>xiBdqklJ2Q5+-Py@WT8gO zzU8SuOaGpBa6nQ2Pa3U~JG z-mHy)d2{g$hNmjDHjeueO^7*2{!On`7=LN>-X{ zd1fU6>AIt#tywB+l<9_BZ{IouhG$!fnVaX*i|YQd$9CqNB|kg#!KRTm%-Fh+i3Nu(;Gr8pV? z9V|`E4$)KsLQd)iG_P%jAvvZizemqZhK>K!cUtVTBn=u&ocH&a%Jkb&pt(I6DWB^d zJOj2u2kI3%)E23ljPYt)*hxZnN&{89mb28lSk0&_lbA~Zp-dFl)SD+ZUuj3J1y-1) zXk^eAt13Heq8ku7EO~Z;OEVMFxt@#zjb?_eC6N3vT!ILI%eXMw3@od^iT*$mQmo?- z!)pkvHrL!?k=V$He*&J(L4^ti<84wa^HRe7@TRfhwGS;pW(_5^WV2AGBASe(C;Z97 zwisLuqg_c!?P@;$8P|rUM$%ykAW@1|g~rTg!V(pv>L0`uSf2+J4PR=b?an$&ngve;O`XTRbjeYLhpwZ-8wq zC^^k6DA?UiL(}ILXmf`ST-WqaU*|US`lSi|6I)F9niKAt+tpnQeXEfZMm@{j6vV4WA%5QB_`R>D2!d zB(6Wzdmer`RBs%xKNdHl+H}xW>yt0Y2jGH&|K?)@E_g>Halhn&`2)0;u)jv641!v? zG?Qrvoq$^ws(p)0cpj?2LM5JuIFJzX9J||PcKxeSKy5?K3=G6-P6u*UCOn8|5t|scC@#S-`D_ zqnTwtVjUn6F5iTF=JC9TLGez&K|biXcQ7!k!1ekh8mfMA3pu&vad z(9mCZr2_-sFJD6fU-$q1rER41>31RMZj;VfA~GGJVk$(MBQUYAFNU9kE@#y(L9$lY+#Lg`K%%p8JC|=TW9a|khf5j+e*{J>?aZH$!g)ez zrV2G~yKc!bR=%e~ypk==3@I7Jm^y&w`;xWTWs%LbQbUN8rCd$@x#aGMkJC3wHB5(w zjt4*g9Lm(sYXn8A>h6LU-LMe+Ce!72kmTTQlM`@V5n4t_OisL0OMdV6QfVXpRBUs% zp?5!W2HUFLe0LP89n&&VD?pY4#t>^>#(;5XO0D>1DnX^EGFI4ZWQxVT5RO)8(~YwA zDqlQ>R?)IH`D?KMZ^!5v#f@{Bw$k6WR9jEvicbe^I=tnXfG;=Y;(G4?zp(IzRDJt; zi1ceF`nd3d?bQDV623chIXmz3Ne~_W?5V-&07qF&EKAhb?#AVudg8BnMTiG-v5Syo zToFGIs{Ndfg^Z;P9j%K^4TGyLA5T~P{yvV7*DlNT%Z#$6|wz>K=XyRM)$w4*UWNelT*kx zyni&Rv#2M0jH$dpQqJT)v+B9y@%SI~_jnW%iG`4pkW&-Nspzz)j%-1T2UgcpyyajI zy2-a$RQ@l3u=P=|hwW=;eaO>IHz#=BgD6-^Cf%69nsF9#fC7IKC-A3He}3f?6fuYV zCOl%(t+9KcA3yG{22w0a zty(mil!+N5GJMcqMvedWCimJe=oy{Ewcm4`AUcS`&*upy^LIkf*HJDlv~!6Tz}|(h zm-9E=Xf&%m9H*Fh$V!zr1I=cW=pVSh0xr7{JT}==8D7*J7hTkqJ=E6Z+fN=}9<%tZ zH>7NMnwwJpT-t8dgb*vKVb%e$3rSZ4o;z;(x`3eSC4p$AB3V=>xtz+QItO>6-=95T z-6vp3m=oyGTz_d^ADEEqO@V8*W!`jNxL9?3#*$#~)PXwsWc#qlA*Q*#0uL_N5UPbhLTg4{s)EWg z7d1mxL`2fmIQwSxt6ygLgvUlORvZG8WDarIQY1=ym!uW2KVT)jmF?)d-Sv=Z>&;=; zz#d`fC5Ke0!i=}DHLKamM9QHngvKM)rPT~<#ApVMJ1Vcn`9aL`tdwfA|6QdXY1dJ^ zmyB}A0gTJ6R+F&P$aAxzIt|elHQCN}B$yhP6iF^|KIKbVS@X5XX-2~<$p%f+bJy`e zCB#59i0le9H)zX*%R8}GaHt$4$y>~TFECO)JPF~o-t5={TI7%4rp_ZZP6@i9awf_P z5Z*`YSCuwtkcOB?s1&D)GREA(J`q`}T-Ov20p{UfLD&brR|WHrO}(oP0QzI|GmNTOp<*mQYn1zlE@J;s;XZV%2lYF7Ry7f z4X}z+W$TX%95cy4B9qda%g#c<>jtNMs}LH_5dFM-SU8KR<>sOAP|jXv%!a+gZy4B` zaIYr{F4mTOTucR@@zlEvj7M5CghrbhXF`bRd>gevLHZ-ARz0esDK)wPrB<}tayb@0 zSFoMOqASVt9jV6bW{YF=SKdif^it@HZ4MhBh2h*>AwEET+@Gqg`%I8xURE9%!vle;_Mw4q7}pcC-}y5n-E$ z4WWn5leSW!DJTDMA-~V`S|Z}6_mwi2tp?LfS~%yOZhBsSyUmot#8Hu|Q_m!KS<;){ zCjUbdvzZ)R)2Qz+v$_&^s$5m6UHX-7+Z}t*&LM@0l`yjE8b$OEPOpXM)Mc(%#nmN3inQzT7@xSCh0RYz zerXz*?d313Vr0(AO=>hH_=Z7XLGPTs2rZc=0gT)Er+m>iCe^$;G=;wXPXAF4r#7z#~WqUvYF|5g^pOi(=4s3wrB_cev1(BeT@KTUr;Ux*46K9dl`B3U?Ak8HdDXB(*fU`lG@C-`3xQL zJtZaaz&zq$gOH!#*^;H{0uXrel`^MrVF^*J1vr4XD1w^+e|UGchgAH$N|AMm);Eb2 zfab1X!^A8^Yz(DclUAQp{kEOLL7?iQPL)bj!04~O<*LMM$y!E~O)6?NVGF|h zytw&;p@V%dZD3d8fzY&j4(Lo>n-Wz!Db}~opyGJnhoL&n@VtNW!-2rVBdIW(b^!RD z)(p3ou+8D9FVlNDNi}5gXdf9i6|foH1&=7hLklabYECOjBG6!Abu?TQN)0s<39Te? zi1X_x=?8kDUQlKkCIVc|g?BJNqc40)9jYd*RUc(kNwk|U%KfQ~?7ydqpa&8d55Fko zVsv{3xw2e!wCXOjArKD!z5^3rI`OCfaSeXX=d+jE?$s{_Q7!Rx`|0OjefK;NR)MONR z+0TEcO|(aok^8rKppfgWy#R}Ztn!7>HE(BLe6g()#dK_0my(AL$}s*O5+1o47fjg` zrSk#^Dt$9fwwY%;X24dsXU{X})z(t!ay`XxfcT=%2x|Tz;&(P1h<0{E@HL~b;}Eze zGqu~-C_M#V&OaSZ#wU-5$mpB8ezmHl^G;eqF;DnYQM zfAe2+yyt%cSn8>N{9kSlO22e}R_7#sWE63%MxC` zg9(O$LTnU^WwnfHksg1K$NqzSRZaBqp(pqoHs! zj+>b8b6GDkmzB}CNrd?T=kfezT zit`M?=#A@gh`eK;wC>OK%f+vsC2t?IxfoYZssTF|6g@&=ZrYuu;9GhlIr@B*ZG6w@ z=fzv6yT5+rMlI`GZh9}DTEhP@#fCI0G(Jh9s%MZ)SV@o?aZHC^=cXTO+emE{8H5(P ztFN5Tp=Znci26n5QWq4td`+SLS}wqtoXOspG%-Z(0xU5^_1007){N8ZpjsYw2PqpV77Z{}hM+j=>lQFU+4s-5z)*4czto zwD{aj|8St({_qVBO!K~X6Jv3Ao24>s5y!b)z!$gAhaD!aBlgy4MWvJk(`FN^6ZvfT z(;3c%_Tv zu8)<}U+Hp!TIn#PVl)ia)Axa|V#6i#>+fiPOgno1UO|U#tKfzEs!wX+mAI(P5vDZ5 zB!$s@kHdhXIu5R)`i&kgwVyfYk;fx@7}!nlC&X-RjIirSQ%P~AR!`JI7#TWWHg@-s z-dt28tAzaL8|q~EZ2&FbbjxT5WSNio3`G#(Vkj-xzYSjF*@%1 zKjz)>B~trPVc>aHwV%t+Xl4RE4?*^u?S{4bG zU4QA})mUkL#ZY(UJq*wOL2St(pf-Pn{-gtDaQHp=Bd>%b!k62l4UHAVWG(fC`xqwZ ze>|PQAU+a5(k-5_&P47ni1uXFl6`7ZyNc|oA{R+KUhRS3!&^42GcTY5-&Bs`4_1068QWv%&&8`y6T557wh)b?B zmU0+}G+K#OTPNN&R!#@rt9a}7j^8926-MV{4W$7e`v|~XFG^9pt$ECjxju9Ii*Yp5 zHQSQF=J~Cf7+;yVEr25QV5=;){2xN>tZLo3Irf2iBXJ^~XA&UlqR*2vuoz6UAsK?@ zCH*DJpCk~3d|zW!PR7SdsJ1guFpV;N;?p(psB(A*LraYdZrsnm44Cha#cF}1TFEzf z=tcQWg`ZYA3g+Gqf?Ucdj7*^!y%La8;cvn&f$`q$x1#Qn1T{3hz0en)58_tM6uP%phs?}|`=_-&Op5xw z-&vPL3ZX?@U^p2>F~H<@1~{RWC;VM}Pe%63xi|Vq=nhCp{(B@T!=YGw6n-U+P}_Js zwM9zNgNf5oJm4M_HJ2@5c-R(;5gp**37D8N;EZTai<;DY162+p@68ypIPTdJNWQ zsk|CiOEZ!V_?BAe+2!DE&4dH3yGt%fDN`}FYwGG9zY%k6gMg1{I=vydzu|KmqcJ$#iNOh2c8M+>Ng{_PR$Z)q~&nl>b$NSNVOnV_2) zUIn{;YqV^~aModGyI_RHr9vfXm0k;cj`x@-(!6a-G|4p-Yiqlh*sL6O&oyiTM`M6r!U#Wgz0hw3UNx7W)f>V(Lnn@kMoHqXBn1U5VOCCULL7i2 zZx)7(5WumC^Rwh*w4%l^DrxZW(P%LDETb&6@$6T~J$seB@;u{_*zx*PYz{W@_9={j zhvp&xxP4Of#&qwU%B(X~NvEbbtm)F)BRTSdXWRL(&ghQ9^x>ppFH7Zy>@dd2Ggwzl zf;XIykv%43Ir-Rm%2QNGYM^`l9bP@H3HpkNQNX=ZT8~ixjw-f}D721bRW|C8Cx-#K zelJ_4udlf?)Bc{^-z`1g*Ovnz{2xyrGbg#11U)8 z2D#JzhBaS*B@aJ7X1@fV0xVH((YskYig^(AV*|ORm224P zTXR1-8_1nV4K;0S2D#fPuTSN{@20ww&@>HwLL&hHq^-2}j`RCgVs2K=Els@|SyCYKs}|o zv#9tRn|G#z?bW+je3Pk}9a2NamW;bKte4lNqFrI79! zqP$76nwS8O<_b%@JhONj%PLDHyDgSVl!@tHh6 z`Rc)b0G}%D1Q|G&ziv0biQ@5AYyD!gbiC zV6D_;!(D0d^Ud^}5A0tJdo7KI1y7WEL4_j}?mP8jY-$`=k<-V;lN+EPk$BiIQGS3M zGwP6oi;Df0PD7g~X|qNkXyPq3S0-U#^Wvni`G`#P%>LkZw~=5yE4eno)1-N1_0#*M zI~^$+KG5X=oFdpw2P!ioWp1OkEcXmm1EO!D)h&UCEL1HtS3CrE4)ZGijE3c6UUo{R zfCBVBXmIi!?#0j8?`&S4h$cBSr-uRYF@|ZQiJsQhw~@yH-S30_ql*cS!&pEDUDH`n z?Btq<2y0}BJ>gAXL3_JQi)G?L8}T=@mUx&ES@a+xxFcvgM8YF zlX3Xd6hE88z{53h74-qyo+gZlza%F6~^ys?KV_k7Ytd>pjDrBE$r`kD(l z)I49bDG}0w@*=E#J3eheF*oehNaz<=Qfz}MX8#60J|4+*$v1+*z$BJc;t`%)3YiZ3 zR#H+1RC-7{C^R}bAF^#BX;Wqp|H0;PFhFB@%l-1Ax{#~imx%ea$vvKo9<~iJX-yYx zhsd?os*5?2HC4izrHia(jQXjm_w+rMD|8r}?_ElXgtPiKPI0knAmi zIWa8kMQ;FcgR!`9m`XzP2J=!{?LVi*UY%@lor0>`f~g`i5@t)upxhZY&3w5!njjo2 zJ(O*^#a%{UG^rt5w|Hyc9#4udfay0Usuvz+ArVGiGR5f$>|}$>Y^>0%Rdd)OyRIS} z>u$2>-;T0-=RfP}dSu+Z(;2Fen!MY9G*9i8EM4(Td*3zQs?R)bgug*Gt*xH@4n2#j zC?K(AXWbIYbYkb}mnO0HjG6kn1E|(Hv|92TJCC#e@WnKq$)@h1hUQGjY%R+%gNE4? z)7lxazkm2*%J7UjsNCwZ(IfYoL~TTKyo5`!2&7`#u0o)Hp5l4ITlG+v76eqf0{J>d zrPz`Q!Sq^2#>7mV-1fxjji#ld@ngo?=wCcqjFS>|IrF6amHd6NDsM$ggZ8`QSlRz#%8CE&!1ali*4RyKNl$1rFSEE}Vk^SJ{uE}nWo^az`(yV2y z?2>uGCqX3hFUR07BjcUf+cv9;;|NC;lr@W7vG6HQgi(GoaoAqGcm?t^1$aa6P+SmsSvn}K#-zR%9udp zy+``<-Yg_&G>j&5m!mj~XYxi&S!v>D>r1wfQxL-~2@r%dfCO-7G5PK~6M~?GWsKRM zCqrEXFY-h1g~N=V#~2596ssgHj5OhonMeRu5agKF>Clp(Js@48o`zh6LYkmPdqmVG< z=9$b^4~Z(qpI^#S3R$}Btm&7>H8d;-32t<*A|B8>sP)WJ{WAES;ayBW-jcv!wvlT!{@oRvVyTSBDBHeMGxms!MfvIY;1(_ z)^0Kpo6K=YtBGw<{i4ms6zx~-g9tf%T;M1qmvwXb4CU6IcSeLv9YI#=LC|8XiFwlvp_DxXt85CkrBJh?9&-ay^u!Jaj5?%E+&C?8RvGx zUa>ZHRpy`?CJQIZ&MB`~N6_sWZ`llz&nP^0Vd3WnjX#HF!g7mi>^xPJDlYGWkrLAi-ngh>w>XNU;oUl}wG` zIJfvcN)q%r{1qfLvz6LtC`hOOCvzXL!iqo1zv0!_*q-t{PYC=WHP!X|BrI zdaSMeCmMQ}1GrZEqy%m2d_WwDWl*O181a--dOZiTRNuN6u2@L_51oO7;wmXBV%l#v zfApldc#um&twbJeyT?e0hF_ir<%C`;LQ@f$bG{rJY*kH)(~LS+zFi`gGVXwS7fqYd zA3V5jT;5eQ{)hLy=2lG;8kcYhpM?f>D5bv%E4Eyvj@5%FEE| zAL7^~1<&Pw%$S{@H5B5vO7M98|K$H<{9I4}3F>{kKymmn!v81ZC+YvVf_#FVyFaFWJqt7a?lN^UU)|wD(h|~x1m#X$aZ6$@msV|^AxzyR{N}i<6<8! zCAi?l+Zb0+$V5sz%`>q?b6A3U4inxHiB1qO4a#g|Fw-DUUbaNWXMOK@ zZ*`1(>LnAzkWxRbq@eo|8h`a_{K=*%#4E?jZ!+^|ymj9+Iv+WZ|CiUlyj%5U!}~DE zE;wV?d|Rb9+q41IAP?q2kgOh$_xTsyXBo(4x8 zpkyncVBshbCcc<-?L>egW06HEN84V9ZjH}A9R=WjN27}(o3h`Ya5&WNLaV8eR^MvI zP<45fBe54&e6?$UX=B&d!KvV7%}aN&g-;|TxOUI(|4_mG;5MS2rA4H(L3|Vo@X%77(HvW9`Eiw^% zpj)R7(41lv>9?{jb%e^nzJ{Qpdf#=0%w=e)w`{&7>o_VOI^kB(r1@!Tlt)Mho3WL! z`JoBvq`$szg`m4M?_fmnU?MY-6wNj~q45Bpg|m1Q7_NVl(D7lYMwjS#N=b04@t%P9 zgv@VZ_(TdvJ#X&V%D<0QI0EH<3YzR73vGS&pa?czrledi?NqGw>cKeh!xzs_Et`6L zUl*ws+MZB4sxYP>081}3@A6Ko zT&xFp$=hBFGppggd^n$N<~*MfkMNrvfCKE(AHAhlatk&@C;y}uk-T(9fxuM2#DYd^ z-mS4(ux92{;xk#B?xXL<@GH)>m6cU<8w3c56rB6`QgFS3e@+`bbaM_bKgCwGTM#yg zWu^0)?JX8{a!ENx@k)IIOoSHWaBY8W!Qclkw($l$#+p>MI^Y-Sp-*@SZ|EQNKjwM2 zfxm~=G$N0#E9)gX34kU_%B1LDpS>amy9;|9M0QnZZM!@lu#XIF+WW^}h;!TiB`^f@ zPg9T|#oG3Y8dFacd>K4r-wJrFivjX}OgfUTaPvww6gELZGig6rc+va_vpb~=ab?t> zrOuR5lW~sqBbg{6m)*RRj-o?h3u0U~>-%_X{C0lY8shnfG;-I?*tGP}qtyH_bqDsw zEE1U9)HKSD;Nb&Eq0w)?YL%UBaXw1XoS>mV>Ens$MI;LDNPf{kF_f7rmf`iZ0-wGp zpyFwP2y(n6p2Kg{)W1S^IPwOUZJhb|IZk=dk{ouwiNaBE_-mkiD*qKPr)k$Wb_O-@ z``4&>xtmys-Wjn$f7ZGatNuf#TCg_IWRmk_tY=TYaD|5)6RnPZYa2jQ>t9UfNdRE4 zEkO0B8Zv#y-!`ai_Fg0(6&5R|p)t9acj2vDFHp|ELNP0NaMhjeC}oj*2S}_z+p@31 zyEhvvy#)l7Ef98t;O%;~*`s}YaT1O=sE={*H)>Hrz3%*_xg{u7Tl{|4Au)xkT$>`D ze&3PcrTu9m!{HmvgSogj@yhTO^tF%!UZYZ&JAp7=1D~@jE2tX6@u8`JmTh%b)B5#I z8rQ1=8zLW?=4F^}01F&M#|odQ?q(%b?tB0dinpTxM<9-S5`KC41dp278HRol1!FFu;;cIQ3THxY_9gAARe#hf!^8P z4k?NPy}kUU>wf9NYU?sYTkTt6t6Hm|s@D2!=g|UX4`8w6;6Z}KEBLEbL{@c zg`Aa<#AiFW6JOsgp~*1mO&@2cb%1fiP}pC1IdY?;X*Xvugp^qR*!ZMaud0C)D){H& z7fe`+$qNI^DMSZ#3UpNC**7bpH_8KrJ=JG4VK#ID+!<6mb{rmSS?qIWz+t;`%xHne z5}EDYZ^Wg8GvSPji&w$6TI#S0#VIVBjjeI#lP{r9TBRg?KGsTom#Ws1y~W3be5uQl z&m~{x4YKB|;1Bv8-%Eo!xh5mAqY6(tI2|n-nlYJye_yEPocm%KLp^yTimx zKi}Sb@FSll{5+jnC5j*10J5plXSzYygLrDcO3X(D=0zjSh3^cIEW@!e$2Fy? z4VJFZLB%4ptP$h(a9-LzcX3cHXZ(<$2}ONpGmm6i-CM@B`2hb)NKwA|KmcjA)&aw% z{U&MNjuHZ)ga9;KDv<*?EcUrx>0TX$C!FVw%rn{iFoV?s^`~rKmLgCV8;|pJeL1ya z5(Wz1UwzwTh_%(7E4gF~kD^Lx=S{yAV~omf0x8+5X58c zWUvsr8}!5vP-?$*UvQsOkN*KDPzNTWHmKF`*SF9qF51Dpp;+?2(F!&B$jI!*6y9=^ zm-QG+v#Xis`L(!O08^%QEDtKmXr-6&`IW>=q^Sa`IXlck)n)WG8rQj4#i15g`~EKJ zoag?kbT=Fd{b)dpq7v@Q(u1ag)ycS5i%@zDi#5_JR03qy;QVc%*d`4Z4V+W4n0yn# z_xLhcKz`tDHBjJimPvO(5nL5}!||#DEUW&@P2Y27)0vR|_9|i4%H#s}y5GbxZWpgk zMdCR}zc(-T0;_RDVG4?_ak^_8k*o#q3H(I5n;J9zM*h^4N>@J>p2&s*fsB-G8$7Fg z-xRg5?x?@M=qzLNcdh_aOrc?R+-#_&aeru9OxT#8=Rp*v-=&$(tg4tuSGAjy-Fi@8 zsJ{tSgnW0*ClfU0KB_@FzrZojkwBB77;tX%PuB&VkHW(iE7cdVM+SlFQXviV=d?UJ zragg`aCL;(1p=S0okVc6dN*RVU#etafT~(v_w^#j{`^N)K(I{NmrmR1LK;4uKe(nP zIEw#bsWM(`?vVDoB56azn5;6N@cUZ7GfJ7z1p|3`o0s(B?8L5ETLj`t3=V3oHxBna zyw%P={d+ds_&>MFrlv;#eK)$*^mmmO>W zBiZZa5Xh-+zjn_j+gn$~U*lupH5?-T!-Yf-mQbHJ=1`v&7HQ(15)LvZ9x0DJHpF094C7gE{}%vwK!?9L z78L+&xJGkRPasL9O}ZMYQxWE_SwZ<&Tyxh73lZ!JMCBGd*J$rfEp4l(jj8=5%NF(l zBsnuRF)QY!S#-)oqD^)*S7+agI|v^DjefZkbjve4M;C=H!Z$*@3mstE(MHPh?Y61W zReg!Z6p-eFB}2@iZ>xq4W9tUP%?Fno45lCDCxt1sVoQ_gq3_&1n)R0wO&*Ks-4|L% zk|jFrEM>kJSR;Y=xFRuEGIv$D{5dj}(`py719`JZJxI5eaZK7uQY3y-aA7R8Fc>r# z4H=9D40aAGy`kZ>x?#JK=?=7JpA^1)ONWqJkX_y14jROWO2dXq!h|YmU9(C&C;$8G z!|y)L&;NXO_QTWpzy1ApXCD^u;kW01KL2F@_}%&W+1U?&d;RtMt5<*L=bwHM&GrWW z`gHaKd-eB3BIoCS{+E#k9faA`l{&w{Bcshr(R*i0KSFR2#dI3Ef+96d?U@$GrUEN& zO_p0bijeVN&E3mdM~!-Nkg3+5kjJ(5ORhD8g}=AjOeoskY8&O2R-&?e6T~7%ohu@@ zysnLn)ML|Q!OOhv;HH~B0a?ft9Oi8q6CY1Mmex^AEs5}LS%7?pIqX&@0=LVeZMAV1 zvyT~LxoN`KvGO@>uE38KtC<+iwQ^SPFMhvox9qC&P!ZRfNPJJP+E9Qyfiy7)2Uy7} z1S!K6IsM}In^O$S>d7A!TQ>!*^i)>}gF=2Zn`Ns$sZZNan@{s$Q@%hitHGz)Ib)Cx9gvNvYYjM*KXYNJMVMuecpJVx87&v zf3ME}e7?WG1Kg;RP0g+d0c;&Vn*J`TL1IvBUh13H9eOvB~x!?4R_yr zaIk5~Lt`Q}jDbVzb6z}$?dD~?da)Z~uE?bzva(~~GH;Y2?*m8z;z~G=?$U%?dh~Hv z^*j8L^IsJn@=i0b3Fdr&Um$`X3NYoiv?J9cLTk3(v$A%tEvwcLCK^IrORC>8sSIkU zAVu5`FwSb`ZrC5kP5=kOH@`^alJ{!l=Ua?q`hPWhhDG%MVGi4ZT#VaK_2oZdGQ@o1 zny!}i|Jm?cT9~*PDhNXXeuSu8?VXl0a<`EReNfKwbaKOXC$r8D88=~`T#6G1ao63H z?wkTC;sky>S?nFKV+w5e*gaI%pXTq-LRg~gwRgI=UOe<*^g=ou3dpc@i9UMZ6JQn2 zWR-KVK1x(aiE4B>juI7IM~Q0mI!aXJC{ayq-%+AEN>mRfQ7M)+5Sc4Q%|6ER{ja-s zlZ`BLMF27#(&Q~QLhHKXOE6*%XKrM2yY1pUvs#N+$}p8)JyM4taRqSsej}!eU@P@? zC7UuOSm6S#_Ou|XDXlfWAZz`+2{*vA8&>CNr3w$%7Gn2HnJ>t-+&b(ZJwL{B37!30 zvt$Ju$#MsZ8z}Mu3;0eG*2l0keg{hwEROnTzXrq@ANU~1GhDgy4r0N-zI^EdLiDp= zUBj@EFYqqK>NU^Fk_XUF^|5~@*d0RXIXHkW?vZ_mYp8kA7^cvd`2LbrTaM=TB{wqc z+jpl1skOSBS9kOId~OOGLpg;pFOEOr2yiWHlkU?SVa){E7prl6S}Jy?e+zx0Gb{DE zfPyy3PY)90XF$AyREvPi9txa_T;7evd&H|k#2k%gcf5cF7D6@dy66TTb8$MBIYf=X3pez3Y-)uhY_-zbsT72j8@6Q?ExL^|nBLD#Wa-R=0dI$c zaEXzVK~&3mW#w*nf-!RPR=kt4e$5penY}4F6wFr5s`a%v#vDINT0q{Ot~5m_5gFmT_Cp&fR9bz zx#*v-3q9Ha=E@BOcmG({)3|?7dZPXqO*^LpZQ08|H;F{Fj)yZH!UFw?e?CEO5JOb@+BEfMTTA4Ao+>*3v3Dy4V}VD%1}F@p10}g7|R@zz%R_gw}Q?jh$kW zFrg5ss|N{Xn)U7FOY)SQ0ZJY$krm|yX4ulv5^Fu^w3Q{iaL20FC{2k8A(2O!_K(Ly z?%@F>?}k7`@zJiJvL<4HwNvEP|3^TGmVI`V~BcB_lr8 zMr}a7CO3MGA80%-75K3WBZZJmMJHDuprRrlOft|{zhy4t(3_v*A$q?)j z)y2yLBLG)!>d-2H#_yiK95TGX?tzgr(d`2a2)+oZI9Sy70WWYEw3uM8U&o$>mJ%l;31Q_W{=#xLLdq) zBctXNFdH4n1xDqt`|$>u(PAT29UPCnG3h_nmJlUWzZJdNj6WzD@qaVJ)!5<>b109Ry&o|W3J_{HxTaxFQ6Ba zoO+fn(N}00W5|0{4x!zzarc zPByg^T7NxMnObiZeNi#MlcU9T$+ASz`Rmu$7u^SeFPR*62EP-U8-HX-c4o_slsDwe zCI_y0y=j(yoSX$WCzMW@0-8TO%P^__t(_D2Ox2KVZiQMk33l-QfUwomw_>-efuOFW39 zN>CJXjz9sjfD3HltLDiO)O9d!{KEI*2tj&pYVqd5A$Q#bEbi=sNM%qkt zJ>o!_k}dzJTjtaAY&M!v2`T2@&!KsB51uc`WvgDy^r-r8@M`r7_~8w!?_?D!-3p`O z+rlm0Uu4yU3%)rQ)r4&Tt}640=EL@E8gyL+dNi)eyzGUCbzN9tqfywoHRMr92|8=9 z!%ntHtneYH+ap;)13wx7(TgmTjTi+k?F1C|9~fem-Ma=_Y%HqY`ZB0imRrzuuQ8mNM9 zMX-u(<*hSkHT4)Grk0WmMimRj-i$L4h&>vuDA#TED|0Uv=Daot;nLws8|BX{h7*uq zJ?WJXc-`bJtH_To%<4zjlt13KHJ0DAZCTKo4Zz&uH4#d|EQcqio+ET+~i+B5p9Csky?n;J<3%FSEwvdh3SP^M3##;EWvxO8lA$R`IOYyD{r_2PN^qW#~qI?#|jZHe$Z_)1im&fS{1-=KS6> zf&n5qBt<}~92NPT7E&;B_1%kbv;)9+LSTFw6>nVK66V?#)z*A>Xk~?&0oyJaID7mA z5D0L0D2d;_m=V5U3;P$`8a4tzH7Mxy8sT_d)mC6;(0pI&Pow(dpLCb?bN@d7lL?oH zzkW0(upfKh;=vXm3T<$8_J6dPmpFXtN0a*n3M_am;^D5uT0biWpggsmD5u|QX9O@t}a<8%hWaUbcnEZB_HQj$W7^*LaTUI@ZIas~Sq$23r9+Somz6&A%z zH>ICPg#G!gko9lva*j0AK3GdaZzZ>dz@eNF=5XgOzhg(XTxqF2q-S4SX5@xbp}AyP z$n1vYro1MW8MAM=`tC(|?Hkny4b+fup%sU@O5g0yW&g+ z%^M3LQ2`~?0R+Jmf(i@clq@TG!vx8SM%5k?Z<~=>F`CaY9|!DdCc`Zx23g*;r8%sa zzqRF7ZJChoUJ$yf8B~nbL%w9AZasoVS1bbaK>)ssR5hP$1%$l=AA<|eOY z9>29O4SSEaN!FMKtU#;88=_PR*}BTX=4>&%a6@komoZB>lva}b4K`>FubV#`*M01j z0A_$!BBX(Ta|qbLUYY};C^&U)+9zAbs4vkPD`-O37}pHuIr(izL@{EjprzVq!2@v& z&y@LMVJ2a5^GBGW?(HF0q`iK-ClQ?u?v3*l1P{ENJPrk$!A@oC7`BDzV4zG?OUlG= ze0x8*DiC=U)Gc7JPI(aQE`g$zt}tI!4EkidZ%iPHZ8BwVHH8bY!$ExMEfi0)OQ6Y( z7O9V4Z@Ia)w#C~^9;!I#UoD+?Y=yyhov-$&C(v~7_ge`^X@~Se1WE=C^u3c_YjI>e zv`?-`5GAyK)q5>!^W6PW^zIq4_xo4+K|sKE#{)^Y{i_?`d3A{al*S~9@r3r|PQu`C zFU%QtA~p`M|AuaEb0XF5&iNa+Fu_aK#yD0?zXw|%lPv7@RW}}c!>fE=Qiy=sl}ye2 zmM4;Q*w;z-Pda@F#A_cg!!Tgh;B0&nuF?t2SaRp=ygo+nV$pN$wI4^<^J;!}y^BS| zwbvYGIpMZI_*}n{Q9&QCDZn=UtHb{emw7bv>=ukS0?J6Wr2gfqCzY~n3Y+8Yuu$p^ zG<1eRHV*77{cs299EITapkf;h9=J%`($Ugo{^efZGA?o^0JW#G{RRiyh;ote3RB3> zN)NtCz;oeR!G*_?ZQLk3cn{TN0E}-;ZI-)q8Fu3GGtgU1VYyr=6jcf{6$6j3YB|opbiJ=9X9$SF8)TeQUM?PpzwC39C3E!@c*U zpajT5yxya)6hc=k#t7ePb%~0axj0UK-$;GZ7ph4xEJK?q9`}sdp*rFg)%fzu6pCq zY4eLpVij;8`moJE>a9n;^{BUYum8$=t0&nU*hME5b&YIZXAj(>UZNq$v2+q6C1?xu zv>*!n?vCyj0&@YlSX(RjJNwqJD;f-`_pi1@kc-wcyWZB3p4?*0F#{M=>OFm`jBi{2 zo@K186AkOcUp@w(=d7l@Q0>qfQWBgh`$gnvLMt()YDj_^IUcmwcWP zz1{1e_}>0CxdLwLIXqA&_ZO>(+rK6m=wlUg``2+V{MXnENfmnUvm3@L_NLMtsp~sj z!8cVpPp9r#zpQYPH2Mk|B$ug>aqg_y9g761?avtiW^(*%w1 zkRl{Xy|fh1Sp_T2_^aY_`!r!At%g{z%Lz5b5?9u6<*~sUV~G?4B~kiUau6MvSg92(ui`ci$~R6jp{m=% zG;^=Y+a$8#ig2k_-}b0Z;rXV2hpfvf!PNFjH>cqhsv@_4Z=_UN|C*?{P3>>A3SIxY zkFGa)2B%i7OV;f=we2ER>=KDaYTXT1wM!(OR{PFXxl2}3l5625Qk+_nwtqdAMsdH8 zLt}}2QBZUXOtoP5pC5q6Nf@1Lk4o$4u~3>1ZDP&zYf=(mCTwY>ec?(99z39~m$tIDu>zw+135t}jVm@moliZWBIaWl~3{c`jVOwO5^S4~n;{ zm?~vzpygoqjl{44#5WtZRgr0b`BHHL|I7nBAGS4`3P~LC>anUUcDP zjEkei&PZ`S%aIk(!yz}}ammfZ^xM-@jO*~cKw6#fVJXvAYp0u#FpPjJP=~Aw_nD}w zjwVean8;E#!mQyuih^#h@JQ-@+gxh*;@c3OL}ysrG%PG)WizFXyzlwUR;;R0*}!r) z%$?feTg+BjPO>Tt4{0yTl#(kfI{;oYmJ~{-S^uiQTwF7XSmM&;&w1{*879srP&EY! z32leZ4(r36DEM#7vfzYCXk0&zZ8&~zaLrBd+8z{wHu>y(E%T{|UHhN|mF{4ECGQ+Q zDm6G{=C63c)K1k9vz$V5i%b#2yL+(f$N&La_u&Wf0I8vU(8^!NuGJ1x>aqs|@ zjNO`1#hI$f9o<=QI_e6sf|1uEL64B^hW)qc9T+QUDfSG|w2)%0cs@qd!EQJtlHO7} zxQY)g`FDu>lnN(!adl~)aJ&V?O#F`JL}eScrKqC>If65gA{>KUf%V%Y1zodBle7^x znH0CIsv)>!E%;BtGa6gqpg_QYUW5A=va&(c!z^Ge`WsxHMkLX;row?UZp~{Ol9$Qt zwh_GEJptK6A1|^}PjYt4iYJP%=d{W;yk?m`N1o7<&!K|?48DtPo)WE+zzM>kt=)OG z@vO8f5xYsU0pagnUB3@bsu2c&shKFl))d}OtR8wURv022S|$#SiHRKRUE(pswO*l` zB}8J_&;wG-1$hy3k7_=F@H+s+{DN*-@q(&hP>6?|R)~O}>uI^)#Kv)CI+Od#V0yi_ zxW~?5SGJ2WVB{$S?NE|R3=L^mmq1JyD5oK zeoc31c6K6)!kBo#V1?7V)v1+v1Hg30Nl#o0ICl@S4C1K}-FZUepmB#ag8#E&Z3z?? z-X81lrKdu5k+$3!KO?lLH?mo8Fu6E2IC)D=${L~eJOnEd9EF&6gc)?LctuOil`9^% zAA+H|Bh4;cRtOe9-pWt>LD#Qee)cswSi$x6cSS5#2y1m>~n+>OY($Iv=zj>zp1ccj{zC>3&4dgcOsv0&rI&U{*1{G6Hp#24oc>z7KPi24WTCE*{R&6acH(lp)2(0Izxs;HpOe zt(pi}<;nd~Jv0thb+1Vq2ddhKp-N(|F_5Y$Xc*}LRSrL3l8*Fn;8X`k1s(OQqn>rt zvr^ld2>0|jP)`$Jo+cx;r$aoYC-x-4J9UAZ5~xO?od!Ug5{QOiodyA$5@?P8c3-<3>r1cGQ7 zRw$Y35d6>}q);-&dqEFPgdI8na%dX-&J?JjRG6Vj5JLw*uB7yB60Fdnn$M)!nF=Lj zK!OsQkq9HS4-k~n#VCYO9~>ynlpKN&8UzMPAUg~lbT4u0p{41=gQ=#008It}>cIP? zStEa8@SnuZcOdA`0f0Yog=R9?PaM^6BG6BQ1Lbi*ejXm+XF9kK4r+w#6kwlG*q+Ec zhCzKIkUh~&*#-1Tfa{4+?1A~DLG>iKU5pJV8Kx&fvY#cC4AB!EPz3Oq0?!leJs*i% z8inSGZGQmnnFh-fo%Wg+pD!fOCGWtkSjPA*Q`4dKkZ%|%6@*VX z7`r_(KBeR+D|9T??J@0ec@*Zr=^st}2gd)Vku=TJQc=Nb6CLQo7;LD>3oUNNRLsM0 zA5KPR9HBf^TCm1YtexUNnG}2o@8q{k=q zxE4fhagcM;;g6>IK*_q|SWX;(OB~>H11GIkkxk~PR&f{PgBaY~NkAnguw1UpI&9p$}_lMS>B*Z==?pSMa=k)qcY@FO6+4QnPBy1uZbsKsWJ?%o#S$Xd=&hV-(j% zdkukxa>3OGSCJb>cZx@&)>cggtkym4OTa`hmb-*>B4n<|IzqASrngr5t3huI&F66q zq;$O7v+j6yCVSFwYO3AowkVaMCB-(p7^aU|U=}2|6UU#m( zdl9pLL1!&^8rYDHfPV`Ds}~8Xl*~AdN5jEf*~tf!w{h$j<%W7$#Lk2TcBi=YFBqwA z?OBVkGIew(a?e=BL%J2Hb2HQR=Oj{ zBt!(^ge6y)FU9iNNH4Vf5us`*7|0WY z%OGUis&-C0QeyP*awsZhWm#9I?FEMI&_YXe(1;Z0cx?h-5U8teaQNhv!;S@r5E-W-Rjowm1Z4w zabA93TOyuy6C5;2M%Nr&TM!TC+SUs&IYo=fbxpM>bQ&JlB%fwyv<8D*+QwMVZ`9jn z67{vr+0EOZXlIV<3Y87ucE?VI<#I(Gq#at~wqB#+R zmEi#WEXHrkl%nhDgJzzyBeiL^RLmUzbf z3e^D`={Y`z5%GwUj}`CQ0uo8xruzP>o{=rhHe8r<1kD3_2u(yJh+XRhq1Oxmh||zO zK&ZEC@>J~&XKFQWUK?#xk&_NXJ~352j49eb36ltqpCI;(c5;! z`(2s1ee^oKZEZ&S=n0cvB+{NR1gRN1^R$=`6j+m@Qj*giV)02vQW^}(oEX{{Bh?em zBK`CeAcn({ZA2NED5Ckt1U*lU0BZ=E2*5tlr;fx;{C8|JWGHZWSjS5+IP7bH!y{mI zyM%QRUC;Y5$rGmj^?pEePsvCNdO~egeA2B6$HUD{Fp=%g`7l+(hP5iVE$cWV zo}|RzBkuX3GV4>%fqse-Oeps!((cURc}n22-b`M)w2(jqQ3A=itC?&6H8Pg<7>;I1 zBoKE6A#tNXd4vW+l^d~Tbtj8$2DbeKqEz*e5r?2_v6GJr2Y}ip-?Q zc=1b=8%Q`<6Gg^yL)8>{5AN@u^Vaf)KaL6IKSyRp@P@#TwJfZkXn!y|18}+O=XjPf zKUF@3{f>?r>^h|1BQcaN-#dCBY}7T)HYPO@e}SDnyj*LFrBtB~B|`awiKpUa!N{Ln zR{D&sRxGRk9YyIjKYeTpTxR&6?5BSpn$(zVf^QvLblvKvaBJrfW2SpZMGpwHc|l$U z;pb6x$I6GaC^|%xku1H5O@th$6(eutwVs-$U^8+B2~F)czR0(GrrNJ8PAb%yJ5RFfzv?jU{Qrpl5X)>loG@j`JX0tOKz83 zVB?IHMYeApC2A1~L=KIEW*s`nG2ETbNWV?Q*@@+~8HBSTL>&(v=hZ(OS}e#*53LQq z8l`I1 zGQHMEY(A<7beU)Wss15VvWiQDLJirLdmH5ptJc^xD_559xJ~WB-35sHp|9wV&d)tg z?sn{g{AFh|vV*a)ewc@8quDlQidrVHk`<&Gp5>B+if6N1p4I0I^8c_(!m<-=O>5o; z7g?WGZA#TsMV^u~cx=3#+bzpcTIcep5XUjIU zBZk(+-cm3}X^>|QKs(`1^k9>`>dP^f>2MVPQ?EZtDz=6_hP&s$9hM*?4H*z?S;|7L zch_abXs%V1x&jVxu)u`$hysETUuGyCEI^Gk(JNWCk#~sq4DSp|JEhDOp<8JrBMce0 ztv*EVf!N3E2!3OyL*MvMxUj5Rk`+`_Fs+A{F36S41B=rMgUF0}_<|VV*9a2^nlsE? z33~)Zvt6JPL7l)*s=Ui?nA9Zb|7__EJGsir>JFOTy-cbwMjt50=Eb}H;Z4vt^LorI$hWY!w#BqZ%phmb_ z$!f{-9JC>90O8~LA5ed_H}QJMbab0V{|Lf zO|Tn5W4Kw63nH2Vl0U*VPeH>#y9{h~W6)e{#!z=BzMG30=y*LCaiE*t9l}d-CG)OE zJU;KEP;2DOepouY)cI&$LF8ypIB4PrVTV)c0Zw66AFprNT@sHveT_Hws(9pVT`?FN z++MKVLb+;7MjF9xnV$BXE8}&{ji=#9y>h=5iq>4MxE|pD^-{i(we=hPI4*P}WTam| zr_{Q7tyTB71g)zv3(y;&sE?bOR$Wtq-3sW2Z54QX&aDf%q=hYPGJZC_-@Dhf-2B;E z5IPMA?NK<``5J-_(4wHX;ZLlRHf5_N+Pg78SCN8&H<$|Ynv0ise+Ok_25%={&VQ1E zk*sJ`&8qQfvdakJJ@vwQhLQq|utReNzS>~~qZEzJiYoHOlP{h;d#*JQ^qo+}W&<;$ z-~DVsK8;e;J$Xhx!M|Uaf4?N3$R|Rs$QAMb)&G!BWUdF4hbwBBAomWyue*_ZQY<7EY< z=#shR5(?Cto}N8BUv#>zyZA-h#ZkJp>z!UhtEt$*E64ePWd#>vcSmFL3ylQrh}{V4`*#}&ml6vGKq1slw%%Rxbr;23n!o`UN^u%W<;<%I2D5^ zcq4<>hU2=!bf)D7+9+mA=8ZEwr6<;Ea$lZ(N#^9~xnAW0w-HRDFPfOB|(Xzg4eq%S@0|^q6yH% zy3cuM^hVy1JBGs!OK-IQSKxE@8}UT{;no(MQ|_oUQmZ|3Vj{VMX?|N`Ix?-;b22A? zl|oj}QR)?duk4Pp(PjiKQ$DJ9?_gFCm<`s6=Z8{vwvD1N?Tx?m!}ys9z6^KJ`o|5N zF&qu3-QXnsD-2sepw=s3(c4e+g59#hZ3J7;z+2>@O{`bgrS7KUQs5E`ZL?KD*Q0wS zYT?i$`yPTErfvfNsn8aHSB&!k4B84V_rYZyT0Guz$CabGyKHLnwi&?6tj&R_zhXJw z7&Jtpp=Hk|ht)~ixSUyMosL^0x*Nm}M8eV^C;34fy3EJ74PuoXvd+g%Q5bdgv#xWy zmj0F)x%$4n-x`T-7@WT_TWE+~Kp#}NL?xhD*)pd_2II;FDDa$#nrF1|rPLPNDsq6@ z((76{aiKuu`<4P>G}~Z^HNN2A$2X3i){|wE-LTqvk*b}3QQ6dFNee16y;1s6Hi6PJ z@N%RGtwM}4Lan6?Ug^bZa-nMdmXKmcy^dZKO3pO9v4r=HaacPDK96fPpsbifk+C)z z$a%iZy!Q_N5ZqXwu`UQ(tih$;TUsaGq5~VLYC*T`WL#l{QPnFCccQgAnmWne=!kr# z1xoLQA&ue0J=txM7AH?;_|~5c9GCajfH2x=&1F4gxjA${@7br)jOqJoTleIGSTkz& z>A2TJ^L?xxluM?>>8Z0`nOwSF1fdp(I(p$K z@%b1a=o2#zOq&1+5NL3og>bjm{1#BBQw)4)4BfDzBo)h>Kz#;iB%JmazBK^i@*V2m zNoOKYu=d0FChr{DdKhwWA|#h-lncK7u0vzSE1Ge!egXCLBXQ#I{dW=nS#)z9$UrZh{vVg;iLzaJpEtQduYMt5`|@;e;QIvT^em-}Y~UJO*t zD%_kV7-Ut+?Shck#^^bARAPR!dHRNpB;iDsni$R4?H=xLyzE+2$g7@Q=1JbN>Xx&+ zCwH>aLp;~qbB-%bJpt6*C%*}S4!?i<^6hhSk>`d5G>Wa70ympl5L)u@Sf$`#z2RCy zYk29`&%clJZ^D`MFznvMBP{9WJasZSy{;4kfkB?N>hojiKpOv=iQ1f zSk)b1NaAyGqn-PQ_5muvvZ=MA3T0DPO>xa?T*wA;tMk$p?c$CPrgLKTRVH$@z2jEy zkT9xu1*fu!uGBJ`g}NRfx@H1$;c{0wvne3h&=gnJyVPukgL96&HhLjLlx+1krX&tE zcfr~m(Ti!@r$cg!;B{BD-X!T|`bN_j>jA`96RkElyPJV3i(m!=JPM=^*#k>59~61_nQ6$jW6Ba#MyT1835j7`=L4Aw$bW#u_NpwH~?-B z-KP;X)Sg0vziK}7aJr=_2Ad@MBuPDn;IJtoS>Wl*I=*BxMAynX_U*26$ImCcbdw78 zEahIYL~1Z;Ab)t++Nrw;C3(@=sx|BOm^9nEl!M0Sm|R2e)T@H3nrDA0WOk$bo@ja= zda~0~K`y?(CNH_Vi5IZN!nIu-9%`%oot`Qb4=lW`i=Sg3OR2(6B|A2UmPd9qk?$14 z?k;tAq3rCKsVOA^cctD(kY2mPjrUvM-Ks5LZY!fFBG17{?_@*nD0)kQr!2^OUOp$U zglaGXuI{}iOw+VXHKi#vV_c#zYX+52tSv4Dd4)#dNZQ;9`&j?}WJXRtu9W^q)GKwe zAeY;+;2E#AOd?jSs&KU|nRy^cldpV1TQv%Urv=;ew?Ov9L&j1lzC zbLDOdl3ecW+cgLF79k`dn4g~H?DmPu>9ZL$0ar>mQMKDos>sQ+lLfi9lF-cWx4SP$ zaV0C)5O8EZJt1eZf)-^|kb+TzyJ#5A4%4L0gP4OUVP+&L)Qlu(6J&33*e5LbOVyaD znw6|FW{jM8OK@})t7~Jf_U#uxD}yIVnh9}I)aA?cak5V!$S(h9!^q`J%iukn^PH=j z;6S-j=NwPsvx_bLNec4nFW077=fBXP1vyVFfqd@9G$>&QvT^B8O~tfak|gN3yZge8 z{+_&~HANW*I8eyX<_<(HzAOP?1|IQ*lzYe%8L<8JYne|FD!hj*%l@lw;S2npY&A9V zMhX@Nsd&d|PV~;5;m7YQUb7%KN;-n{5|6=~b~OiY$d&gU^BXRL02PZVRzRXsu?NSHbu?LNO@sc@_K>&lkWmqjC=b?e_#G`{1ZhbWiyueimDf8@%D$ zcbE4%tB3uccRPbVxC)GUk6ajD1hn-Whr&q{Sp;yJY5lWL*zO@igs(|pV2x5 z@z712v2wFY%N5qQes$fon7&};hJ1DHBy7)L7_uWkvlZG`PZcTniUp8qlkgFx;I*vM zaS>1L6MA;v*|{LEy>BVWWKpmzY88+x90*5pKy@?GmTGbHM^N63n)U=I;K{pJ7cXDG zT5R**Y~ITXbmTwu0~{vsJaFeANm*|KhL6^a@vGshyAf` zpXrYFb?5t-_6Q5JmJovmSbi&6>9tvP*AlBU7=$N{V$}j){kaTo?4X0jzGK1a!b3y* zj;)49)_m5$DoN2!0;!yJ&mu}3q?Sx?*~(_E~vNG!$Wd+&K8KZEr5S?c{s=jJ_zGfl=jTn=gZaw#ES-Jo|z zMKM!V)7Tv#n~PR-%W77&SYeqI!qiZDTQYH7)9fbb6*Gbt~UcY_Q!ajAM z1g+5xXD38B+{}Od{qGm#C9fd)m>Xmk8$PEej=bz`Op!Z3B4p|W!wt6H$7)s5B>4M$!; zeu;8vr&u`V7QIVby7Tk4qHR50EKW`4W<^v@E3@bI_YYb9=rt`B)*8`5EL8K60AkZ- zjZiaEpGDOn(?YX0AG83@HgYTvH0*S1PUW`;p2~O)oXw`BVZr>v*gtR+$Uf+ijMpO^ z%y>;Xo-bEH5M*y$yG+h(cweBhybh8A6DUqOk>g>`cs=mBjMssuGhV4@G+t>ZHC{0f zA70&(*v8F=-8U@N&uL!`C!Fs!Q$m@WmWeFLYr~XrbJ3-eMN=d3l*RD6%hOw_&z8mk zX6GnP$t!-#Dn-uz5=v4xKs=+K|0MGS zyeU0U8E&_%S$JEnZ8r82yBJeZorPtg*Bhi`0xR6frZPS+TR;g)pl0=0%4>U3MJl$| zDoEvl>>_m4y~A`53yp4FF5}V2!9Jn*qi6J%=T^1>7k3MjT2f`KiQp<%N@hF=#Go&4 z+EB#;>8Mkz*0~{5Qv)Sb`V2bXlg3)eoLx1`f~(E-WTXGk!x((!RQ4EfEOFk30o%V~ zLUE8w^y#8yCoZ9Sm!5{$`1Ci7gq*&DnHuBzr#Ef;zl*1MYFSY~@vSKRXGlhiLiLd^ zEZYyu=$a&`u(rM>0J}>&A`rTPQiZFsIoXM#yz&cDSOPPsg>T-|A_Ih5TQRc%Ld)*x zZ28qYTMmz+ZV|Gm2XYca#cD5m5ZMFk@Nm2-d4w0WsdM%jR4DDY2;>^Ax&s@Vn*q+e zZI)o6b{lt}F}6Yh$OHIg0HFmMEVBlH3V|Q%Icwt;;>C#*mTp3V1hn%|u{M`?;*M7N z#nov3SXU%#@ArjL`!&(u(W;kEA_`z5_#Lk6~5YF4)A$qFusK`&Rxd`DE| z6g$AFhL*A*)uU7G=v4c4IMpuEsisd-sK*Mmk34>30jI)Nx~YDPmo2A5@wT7#jV+glj%a7`gH$pZqkA_~kvpHJHq5(K&3hZhJtQ0)mT&CFb> zWnr_SK(Z`p#)7z8?d1;E1oLwGKyT z12qBhYLz^TiHZnbhp8iY9pXlQGrpX}j%ac8} zB}4@C{%4WI@sjS07nk5P;)GFW&65wZeT^ETiMnm{ zeQvq##*MVqn-0HAqE1NQ=1jy9{(I7H~v zGc!*fWSid7ZA)EpI37P8<8zxKI5=wFfRDE3P#+Q3za&&Sb|J%(vv91xE@`=DWzP4J z;Kg6n`Jn}Kr?l%6@ z8kuHnI49*3M$SJvk^s{2s%>|H;_}Z+YCaP$g)EGo1(UZ4TyYODyFo* z6bSo*MF=w5Bl1Qk{WI1qWJp-t;Qf~HdEyIuW3GLI4&AvpIL1d2IiIuJbL5sZqD6tLz9&?hqRlST_k^bu^ znQVX&CWG4QuEO{6Dvt0@NEzJXW!j8y|KE@C_L($xStb{nX#tTci?I%!9X-;uRN^#nc$d2=pt8Tq=xFK{TWNpC$aINPa#Vg~D9d-1%1NuBiaa}LXfz!n8 zx8yBec2ITL(h!*!U)rV#kGBkKaNruFo^fT6|5{}tCbS1^#*K`T+K7rR+wL`(W|4#a zs^=GYuk*A8sQz{*6I96)_>H4 z-69BvB}viJ*4Q-m1NQ=HTM4#`OK3qptL<2`6Uy$j{5FSZrJhQ&X2SHH3SVBjil$<| z?`+43=tDk{gGwN_W_btuHjx!{;o+nRQ*qy!gve`J4$bH|9kAps{i2~Gc6q!CkB(Bm zkUB`qh)^~DbrTV!k1%LJG7o6Z@$&HEQbCL~wTx**nr@&sHnLQD-|DT(gSmrp=|?iM zKDJ@wSs{o7k;2@2Ifm|mp5^)}aTH(gx>zS?_I*>S90qpV`$Cu&VL@CB^ zcqQWb<6*eJImXjx$X1Wan@iaJnMT!V!OHeT%$C3Z^Ecw3ndZA+wJ6D`BkR}Wy#ydb z-M1MHk{J-i9ojk^5OO^Nk|6W;aA9xxce3Jk?#0lTzc)+|pT{h|HXGfwl?jNZ?)HKG zHiTY=Gnr+i`3r+~z-&ju1PQp85<;eyHASSt)ZE_SG35>43r68}~GKcnW^Gh#ICl|kc5KZ^bWC!GYkO=d= z96TS7R!)PXqL4hrG1qMjwfF~+J!~YIyT*(@2+)p(!s^GVt#&2y?3eGW1PN+h5;6J= z4r=K#W~j^zTlF_hg=75CwbB5Y3FqN~v@HzDU;Y8M18Tr6{UD0+6hv1M8te)$C&se6 z2~T~TY(dpA$TftgzRvJhj^vT)W7$(Al-OBXl!(F;Mk5>&hIutXK1lx?fl|}?`)|<9 zL@g@!;5?+Bj7Wi`(3A`VJ5jtKtK+|$A@)$w$;YG$Ubb`-?Z!;>h*8b}BC%G95LxTu z)tPh+02y%7&+Qh*mgzF_a@1vLwR9X;1qRcjq+o{cS73Bp9ImKs(BlP68cV5Fs|dh0 z#yd7ddwEC|j&vsrZWy*2d%u4Lrv^X~BP+Z{QPR12k#3!81Ll@^97-92{UIJm+t!Srye-KshslZ%BcbE{gDS;n)( zPQ&`f@Hno`xerN69~l}owe~kFke;U`6#^aV0>KrcHRW$kcXDE$2EsU$3eN4H2Fc7X zTvoZXPKtJ8ZRCyYk(02&OH+V_G-+&7r55Wm9!bCHhPPV=78|(x7+Vs}6m&f_ke~$Ks$he{z`406GRYWKcX7nxk62 zKX(<}v95A8W{SUD@*whS`d_PBHaFlz-YuywahQ&|5||>>Tu7%463v03_?EB%RZB}5 z*b!V;&*u1fR~2ofq{5!-CyAkv?B)z>^yH!vb)D@TKU8LjS-3Udfr6JaJ!Q05FhCW3 z-h-2-;tin>I2J|b%)fDb+m6Xr#ZmT|OI!Yd{`sejNTRjVSNHGTy*+CFoA5IAphgq& zxx~6#8zmuUi|R-i^J%R?SgMnc6dLM6BJLt+Vhai~=CfM_p8lrZ(3B!mcSJzPhTyu) z<$5rV$Gn<+a1>HsB7RXst$+9zSO>xK-SNOGokcG|0Zoz@IzyP2lT%u$-Ya!ZR_6?H z-D=kFrWiKDIysSvVk|UP1U9U;b(`6J*3@FFMvxO!gUKgnhL*@Sg@q@pu2m98)R8-8 zD#0z~js;J-Cj?oI3yBC{p%9r54`;K{5QM|}VbcGQa>z!V4~5nPq69LLafqCo=_2@J z!0#&db_;X8L4~v=W6jL}rn}LlJ%(5^8w|raDHmhCpq=D3CfjzZL@5j~{&c&>J%Gqy z-7|@pboV=AiYlSUIHXBlk$<0*>lsup#%NsGJ*{!P*lZ@81kJ@2a`jAGHMcf{R94d7 zQ?x@l3sWLw$(}0^B5LtplBfuKd>HiF^D>wY=AW#JwAOLvKZzF#vcI;|ia<0F!!&q% z#g#-&^&?Fy9ms;r9YWeqCrRdli|ndldlQ{ehD7o?17i5f^l=_b)BQ0Xu#INOhf!-=^;Bor8bI#NclC9BQ{b@VP(8&( z0HDSbjA;VNAHQu5dv6rY8h}pL!90<=6hcZ`a0`>BZ9|^eqF{M{qa_GuTUE}f`*#`j z6<}^y5qY~?e>wjcWSa2Y6+@;j^lqg-OeXy&<5P_#8{j;4B+ZY46^f~J`T z-&qNDJU6rXb2xhN-80fOA)^;sa_$Dw;@R~`KAE%gnEl&BHwq_fQQvnNPdF%CB~?a@ zJx=j~Fu)dil?SV#f*7x)JCMu-yLE1JK`oyb>4aC1yvATnT8@;q^#jDUXbtrGR+145 z5bh(>+lX@e`SZ?ujqeiNSrla*WnM5a56_k~BY82VNDtC5zFUjm@xHS9N*nf#K;q7w zBGu<){F*hSgJc*Ndy_ezWqf|z{W?_K5cv)8sY#QB(cbe&u5u+FUsM0rvLRppuoPzD4)5!EBL;q z+wMoI*JWNc!Gsu`KNRTqCEPMpHFxFHLw=)8Nu(Wyw|g!o6Sn`OyRbX*5vH^DH+L9F zRUJY8uhX!<2wRNL>xzN3ALdr;{qs#G33|)U4@QgFHgRa-zcn-s?|c(3&qY&7{+3N1 zv>GGOlyzxb>CoXou;b54ek2Hd0!)Y=9=I zOkm!;!m+L2jfJVh>sz1_j1Jv!c0%A8)N21?b@Ud!Pqb3k=QmUpxmlY|&gGa82|1lI zmUGiU$&dc>6a3kGa?nHfFwaBc#Wu!NH%GTANK#AMdF@NX+NtMSn+fmwLfA)uN>6!DFaO<3|Ms6M-L7 zjw=vvyuD?Y21t!hLJM5kkL^xge?Ng^l}39mwN6i}^!zdOMr5Ao^x{jO_;}-dTWy?% zrM518DYb}2BzK0DPv^oAS#x=*$PLz(yee>FVAZ`B1@Qf6iKVAei@~p@*@NsyzGMj` zA!x`Kg@{su2W1wjTJsk#PzTm(yCW~{HBrA*mr-o1tWsWIwA}ix{Fshe9R+PzR{|=a zGm(_jR-TbTi=5I)yZz(Nx(emkO51E0qIR#bw(gh>z}*8wnh&5jZS4Nx>LLQbdWM9z z=598)?X+qtlaf^WS9qMzVw5o_9X95dJjL+JmZwWRUX2Rt=dcq4-RAAQd?dF3Fw~2| zQuvs|M80RmA7<|k?2|}iqU2PqrUNUFY8(5L{e&pe>ao58Dd8#GRof?52u)3Alji`= z8`nH-G3$U+SS)>QT@jvy=PaWogHQ-h(=>K?Lylo~DM{5J+_4e8W><-{K%B^PueHvx z0cDJI*PdloOr6!#1iPraHxc-?RL~N8aK}*#yI7GyOA&thr1-5uG=oXoVzrgaXls_L zVLn&-TI_REJ!8b_`%qgjyn%z!lyKVfMDI%gC9~a`MfPYebkjc2Q6;v*l8V-k8yy;| zGqirX1Uqvrmk|P0<7i8Vu$dV)z=PagoXu=k0AeWDAWU036xJhVgHVqhm`#`-J3v~) zx481hML*vAoVg=J(!2R*)>>|ctOG#HdjxCYteoH zUU<4hy*O7^qfordKSF}Y68_%`yx|RpI>>ccYVDqo^E#!RWN_mhqn?v=J77Yoa`%)- zlB@;eE92}e*1|S}5*mMaMv+P2Il5`X=ae#j%lwvps7eOa?~Orbh&I3VkHtcxd(WHM zKc^>;6Z~->uhL}MzzdEn?q~Zq@o1J5(_VPF#{Q?@5Ecemy`~zB21HoK0!Lf(oSJP` z?JZ-$@NI4I8M=hV59_9~wP<)lQq>oib+T9V(oLGG;fz}JLTe>@&f2X!#11;bH>jqk zE3HhdK-G)hoq}s^P7u|Qb9z0++soUsEyOgrx|MCq2?-}AwzolI{p}=&V@!HiihOCt z3G@(Qa^T%LBOb!A1Q7u-tlc=$4yEqfxTS}4}0*P^9FdP?}>gg~q-U?-ics9DP+8#XH z!lm6(0_gLo#jh>ToBe@h#Vk*c+T-M!==Z3T1Bq<_4@am0QXKhUOLiy(pIx_~&%XDd zS^vS6j8j#8CWMJa&}|N^J(4E}eR(ceYx-fdsf=I&;w}9JB$k5zm<8x)xA3tZk651x zZcMny0(7OO_?^4H6>Ub1dLz!JB9SZVDAP)?e}4#y=fuCUjdX?*`@3m|)r$39Ftx1M zjrYx+?p`axw|-V#yHfgd-`-~SWvP)N$ATVPNQvHvbXMWPo#*HbD3z~80Qj}f-kHp- zET!bRw~WX(`x}7D?54Aw+aRB?>UEeTy44|8cu0SO^xKg!k3sP6MY&>2HtypiCpyA> z5&eSzi2~tJ$k-T!>JN(n?ISSmpcYaly(BOO>uacV0o3$TZxs1A4nx>@74*~K1gnyZ_weBiullNskO`Ar3AB)RW)&ZY@_|+R4+~eFJ^0XE&G4yE_ zR@48ZaU-j7DA;(erbj`ahVL2FR*z`cVTsH$DyO6>Y_-Qqr3hz9jYYsB-ge4zhA-SV z)N=R}_Fajd$}QAA=6gT;dLKE4USx>jVHDNiu zzCQAU{6J9<10oM&>X!pMsNi~Tj*a>oysI>M)*rz{7b%X4@n@1me^oJ~hO?PMxl zqs?;~&+hjeJy>y-g7WnsT+wf8)D1|aM_`yr{2IowD$OjD4IhGOjZy> zx`vI=N30cSFc&rZ)3-?>IeaockS`oT(nxJwVRhkP z``KK+7nQ)%s3PTw3@;KZDC7t2=JsKqKUAL4VC*~CVG+z{(rA zA$aQjH;-`n4CRGgYOs3{j6)r#E0wwk^S+v2gW(C-Rm(d5b0nJ)uy3djC(D{pk*qEu z1$Dg_e29oD>nS0^ahhoSP_=aOp|j=3p4~~3h$Este29q{!AIH^udnZBNIW=H59m~p z7>rsjzbbmNvt~nI6t3d9s19w@H+ktB?`pjih%DBGwVwQpW7`9_stiJNTaB}y-xro$ zHa9T0+^Bq_LWa)%^gkP3?I82V!S*zu7`7p2>eZvhSscmbjbL>A%`xp`iFI9GC?z_n ziv(-g^fu#rUlQ0}Q2v7)E4%lDki8xY{`{Q6|mJNs;` zB7azVXwo4r1}(MGv;P$TMk0LQ^q3xQPAQb2lh*X>`2%|z7X1z$K{9V4Gjs1_3Aaaf z!Rqg%f;1Z|)69js8|MHO(MwR+;I&=sD;FjLGUhJ$xq9HB`Zqh0<++@K0qH`Tk7D9L z;kH+_M92g&2t-Ec70-X;rbwNc3cKxrQNH0UW}rmh3NKqA~3S4GrDrIJ}3D8H}E@Gm+W?oUQAc zkGs9tr|v_2+3bkopp)RXzdv+;&o0xp?g9$|&MI2yv;`Y8P44H=__X(3lHb_nGf9t# zCja80HHCmYZLGbY#9jsIcDDc9rZV$Al|cdn@8)IzOijf0LC@g@EdILoMvofy<+wM3 zTGGKoO2tgq?tURCi;k==c_QiqbbA1uWW)n}SNo6wpONEreR9pTkuvAWK`Vd@t@$Rg>vdi$D2WIKGX_y{4i%&f4{Y$mka@O4fWx_{%n*dO(Ivk%-4 zwpDDcNqUE>ZSRIH)zGQC3k||aPFSw>f*$R@IN}_jWCFMmXFS*%w0RhMNUGLr%41oh!aH7QxDI zwn(P^uFeu}^e;G87A(Gr48Ah8l0`ig4P=WL7?U-+8-^|>xj9saYL-U@0XMrm%n}Tu z&X8~35mR@mb3r)gzk_lE8>$Z&be_EX1dYEOcO4Ju>WJAb5fTUqk0%EQgAYASQxpw8 z`DPx@UAsflHIk|%(MJgY;27I6{#9J1BfMR{{@`kW*AGX-AoAzFpC22VAu(Uh4~j_) zGDJ(jj&|;3Lfy@1lJIqa#tfcd# z!(g`?d&-DR!k4chDY~dWNPfFt=f*D^A^`u-p}M-+Y4i36$31c-4gdl{_frtOn7W+~ z`b!v#%<28sRbPPn-N=$Avm^Evt5M<~I=WP)DBoD2R^aT)X;YL|dG2Y)1%&tWaobVl z#{KwJTY;N^`&*TwNvA7Ryx9%Hxl((I8XFA=lIBc;bv7P0^Q-tZHJeLL9LtNbgNy4frLN{D)|c5!?y{!Wrh0(N zH}1)QDGkL%Jq7oZ%c;f=Dx1#gk0##4$ilpPU|Fkg08!nxa{DwO#c(ag8!W8lRQXGm z>(y(_EslnKLsh22rM(W(5WYDjVIcE4&(?*#_iF9Qvsk+*B>8vsGlAp@-U-gnIr(D* zy2{dwU)qb$CL7iP;7Z+uOsmV6FtLVDb>#gxZJtGd*~&j2G*`}&0gBd-5$tC-zG_qa zB#ATpm&fn_e#>SsqARQ^NRoQstZiFM^({PHOie(mYW$PYppoERuXiK0y4>lKDFL=7 z60NKO!WQSG15IHV9uXU4 z;)_{nFyKlqujF}FsmOs6Ql&h*lK6BnYl?1G+d;d$w`W-a^c`)j`SxL_!YhOxsga{J z)~k`mI8Ssz`tm55);io)a%K70cxjY|Jw9P=W14+$TQ@Kup-1!w7mTREN*_0Kg4xMm zzIPtfCHYz%dSMvV4N?unB*yNNpL16$H(9)uPxT?8_7pzNQtxiF6XqyCvDW`p7{rm% zi*z0yu~OSlr5+&E?ZWgK@_@KBkHHE4kRhC=vBL7;aAclF>pgHp!yd$;c>yPrgPcDY z>LjlVcNs}ReYJnI`5w!G06cn8H!Vvuh%);PF*D?5aJHsX9mZey-UH*tkxuumz1|0v z`>d;AtcVANVSwib8?sMS94NfTP0KrnNz?=dehanmg*8w0+P_$><1vwgjkOECUOf_^ zroQ$z;UUfORYG>lr*AxJqf3ZAH+a$;R|yizt!1vRc@@Mh%Pvsnr}_<#IqwQ671e%F z?g^%FnYyqxK%s&2h}INSo2^fUzw-}0Tw`;*(U?f;bZynab!(Y>ZFT+5K%Ij9=U{cK zgedU1B7$kojKRJF1p8}j*K>@*=jTAid+cV}LFVY-FDca_r;+iQCEZ9+=)~1XlPp+y zhuCUZ2Tmh0s5#iqA3ww-gv8buf8B=-kYt#(QJZn?p?~=vIzJ(Ivsmu zFK1iJ42*qr@mc1-a0|F3Z#G(+R1<+74Ra3Yw-kKuu!fMRj&Dwt0A)Uq=>d2k@D=u- zR0S!*^`><6CV>D^`RLzGE2pk9iN?%ELMOe6^=1Y7@5+o_2@!K2q+fEzgikl2 zi(L~Svx67C`AcR^jok^sYab%La-~Er`Qa>12rF{Yo%xg+)xMu z@JE_KWKLVcDJ#DdpR0tMn%Bl2rr&SXr5sD|(A8%73smOSajyp1oS2H6Ez5$w$XR2> z?nIyfRpr*|BvxDx-QXZVZU8*?naZqMvz8F1t)5YD9?__mhb4TeQEGB2LyIFQjV~{u z%vP!IauFzv{hBv&pTV8znNgs_^M#5hG4J@!Z`OsH+49pI(byYH}@2&5ZB z?(yo$|IXcaHFMcM`&$8mwoIi=g}OPWa>39^Z{dEP-s>&kgj z6El*AVOeUVYRx+&9y@k0DLOmV%eknw6MYX$K1JWGAM`xQfJb+7PAA89C1S}^NZ!L@ zjU)t7zWs7$&{@Cv_QYG^YRfmP(uf-g4Y{WifA-crmOC{C&3>@c%k24HMw z48hsrY_XLO!P%mgeki4>$5^(cU6Ms@N#jToUhYL5NvBBSej|ZFKukQaHjbK#Ukwg1 zD?<&f#bO~{#p-fwa`!ObkuvL6ILvcA-nLBL!!u%W2sLIZwuWHt_cKw5P{#<7S`+Ju zb`Fb&#v61jpJ;9MWriq~n9l%P-xA(X`yvXKm>Odf|yGnBtRXFMQv#HGG5;+vH4Qizx&thF?M|JAM-9`p>%f+ z<)QH3CAU)R6-$u7eD9ZM8hRB_1By8QolP!ktZZjgH$@#p_HqtFF8wZ}XHRzoN5Z`U zG`$Y>Tg6xW0{uudk=j~5zP{PIvaGGLii^78;5?h0I?NAYi&rKo88ymaV;`a7L5o5r zLwabcCVr!r@yNloUD`zUg{;-Wbx2cdGqKI0x6FJRt(lI)V9J)ZDAI1#5NBKr5gLqb z#Y)JRkksY0cL^N+jc8t9*#iqFT|9%Pc1f!KN+QAm;wZv4{fWAAO{Or_a*8aJIuYz* z!aercYi0MQZRSJIGTW8ZA}f}b!AA8|LaQm3hHa@YQ|ul1r;Q;?rP4u$e2ku@K|=;G z#=uhOP7(Dtkb;ue&>`DG?NhHw!e$jstckkaW$~v$T8419bGyY9ZL-5D3|%+RqP(gv zS5ciryS8OK*W`E=WP`g4Ykq!7P5c;RgN<%&dh-rCC2J-Szks41jn6vt?66$#`e;cbe=;asqxG>!Q>Fw$I==ph{AAs@r5A^GCZn3Ya zx=#wt`}JPwi>i)7I9#sdFJY%2*Rq6sLusb%giYCrddm<PF(MD@fAnDw-U|oq@ z8Bw##95?W#Ii*)pywdnxn|xuFn#0Li%JJiby7le}zx4kqP4t0B_UiwOp-e!k^1BF# zWf4=`^0ZpV2ekd_D3!1JRCBUe@#sylv?*;1Zw|p@V1&A2q3)f-916;H3X=Kk_8f zXuGa;5dN5!E!g8AXlRn{7V?-)5#P_x)P(>x-}M;+a3fQlBvL_#{!PWH2F+Tf_kUbw z6c@5HOpAW9I!SZ83hXpIA3DjwDy0*a_2XIJNRKpX7`3#WPs%;)Q93&G7~D{}Z$J0S zk0QEuH06w$pL7&sr3kGUzV4-t7^u3(*HlRsM6bIY)IzeS5ju>Vyhv3;f*Xu8&yG)H z3@`n;A4o=!41Pu-gdGQ^*xE(c`QzXF5Lwr~Q=ti8AH`*+5YL)|6m8F6K`>W>ux1hLW)3b%xP%W_4oG>99AW z4EJ6Fc{tG35dJA{P7ESvOWa#IO%pH@oO*i@dcG!gYII4FFQv5PPWY!P(4{975cB#1DYuhvA~NX z*aVWP;h@l(Hm8Uv=S{w7O0uonY&F^2d>Y+7wDrkv zAlP$ubPH^b5?YfrcgDOfS$v8~^~V(^GBQr#Ju{g~ zl^N^fzk4SYnj=m-&Z1cIV^Snt9J)z+vdF(?7mHtgV6+mG4CC=9`NnDguS@;@0b>iM zf1p3X-yah=6%M#Bv`79*!%1V=Z1sZu^WL>8X*FOe^bSwc=OA>#{(ikZ;S%}O|w`l8l4?2v_WS(@$Zl5?+;EMwJaxTr5~VSTgHWdXJdLA^|q*^D#B-uAuB zh=opj^1eA$U`brsfZXt_r%!jWx#iX+NXgru|5BX90v_%0Fy> zLcf0sI#dvkQ5rxaqr3iV1{Tp8KAO39>LcQ&ETmnQABzkw6VGIbY68((pkO`WbP0;I z)YY76v(+&0yzbB2`(9^q^zPw6!aJnq9rQ3;Oo(Gm($G&!4(tZt?!eV6eVQsqKYr!X zs=Zk#DNLtD&%{T=Gvhxp{xvHwM7AgZZr&%p#!m7nO^mp-oJxJy)YHC7Gjmbsv zz)SZe$hFT^j&flh^O4$igIEhO7+sBbqA-=~%wXBzu`n33k!W+xA)_wa9BI6Dh3-vk zyhS;|?}aqP(~AaE8!!gpr=4iMMcb?u0zh9zxB8 zhL)p@oMKm)X0m6MiYU!?dkF!U`XrGYlz5k~@E@E@#zkXs5;oIR)|KuBNiE0AYzh*z zlk939DXJ9DDLy~ik42pacm%gSCSH3K3BD;J@9_zkv7ePc>>mj!G3D7TExV0!8xazh zO-LY!LOB@*d@?%J$!M^QprJtAD+{m0Yr`U zGxHvvJ!U9t(Zi?(bnUPz74rjhhs-uzh_?K+P&j#Z+-8i9er4?52MngwXh9ckO=)ep z`@zq%uueKImyYiCW_hlRFXrqj%b9nFNM_cP-#Et>fe(EWvrOr9=>cX@cPgpJbsrwg zBQ^dIgIw!L2L}xA)>WHU)%ZqG<#Lp-fTn|nO>T?A5ACVA?PtSVyYB@RAzR>0=Z4g) z7w7}uYj71R=Sv!#Pl9Cl;zl;pb}u_0WR28Hx<0ZSwf!FS_ErA!YG!cBhUdi$nfwqa zE7#Ts2b98+tLwHKyoQleV0BzEK73TC0MW|p&`)q2WH>7yfw%!Ae`6@yg@*`{ETEkz zpTUo;^R6MXjr|whp%~h5cRfsHTk7_D9-0pai7>UBi(qwfI-An==E-d!E@V&AyTD4d z56j#kFUt9xJ$J@-4ij7_x1p-oYUE4TevHnPZ0^ZrBF0Jd&YV?hZHGl>9i4}woZwW6 z7yE`J9EagaA2Yq3q=ev5UHj3)Ry*YPoX?zekOlM zNxK~QeRxlBBsj`VX$??214fHZ*8)8Gf5fi;oVk;zC^Cz%%OiEqM25O6NZ`>c*6DP( zi8b)#8~c@q_{ST^yx6wvk8qcR?R&Aux!umJFF)+vqIz>dPKYw9?j(%X<>`@Si>>{63X`W1rz(!PT*AT@$XB zS;Bhy`Da7?bn1arHTbRbsGwk(;L~=v>91x7GvkJ?KOoKEqJg2zg;GD&E4QqpGoTn; zt=lw-&zm+E^#~4-XH(jgQhXirkhkbd?9HOggQc;^QnzIkz5vImrL%jCtuuTlE%zDm zG9SpA$U2|?R}2a&qrrqsQ?O)WxW-;EFLdve(?hz>0-=;*aUW&Mdl@{a4=hTUb*n%K za1I + echo "Applying CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl replace -Rf /etc/crd || kubectl create -Rf /etc/crd; + echo "Done!" + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-delete + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ .Chart.Name }}-delete + labels: + app: {{ .Chart.Name }} + spec: + serviceAccountName: {{ .Chart.Name }}-manager + securityContext: + runAsNonRoot: false + runAsUser: 0 + containers: + - name: delete-crds + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + echo "Deleting CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl delete --ignore-not-found=true -Rf /etc/crd; + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/manifest.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/manifest.yaml new file mode 100644 index 0000000000..8dc9dfb447 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/manifest.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Chart.Name }}-manifest + namespace: {{ .Release.Namespace }} +data: + crd-manifest.tgz.b64: + {{- .Files.Get "files/crd-manifest.tgz" | b64enc | indent 4 }} diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rbac.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rbac.yaml new file mode 100644 index 0000000000..a4d498b0fa --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rbac.yaml @@ -0,0 +1,76 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: ['create', 'get', 'patch', 'delete', 'update', 'list'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-manager +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-manager +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/validate-psp-install.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/values.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/values.yaml new file mode 100644 index 0000000000..99e63600c4 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3.up57.0.3/values.yaml @@ -0,0 +1,17 @@ +# Default values for rancher-monitoring-crd. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +image: + repository: rancher/shell + tag: v0.2.1 + +nodeSelector: {} + +tolerations: [] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/.editorconfig b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/.editorconfig new file mode 100644 index 0000000000..f5ee2f4610 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[files/dashboards/*.json] +indent_size = 2 +indent_style = space \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/.helmignore new file mode 100644 index 0000000000..9bdbec92b4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/.helmignore @@ -0,0 +1,29 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# helm/charts +OWNERS +hack/ +ci/ +kube-prometheus-*.tgz + +unittests/ +files/dashboards/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/CHANGELOG.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/CHANGELOG.md new file mode 100644 index 0000000000..8178169b91 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog +All notable changes from the upstream Prometheus Operator chart will be added to this file. + +## [Package Version 00] - 2020-07-19 +### Added +- Added [Prometheus Adapter](https://github.com/helm/charts/tree/master/stable/prometheus-adapter) as a dependency to the upstream Prometheus Operator chart to allow users to expose custom metrics from the default Prometheus instance deployed by this chart +- Remove `prometheus-operator/cleanup-crds.yaml` and `prometheus-operator/crds.yaml` from the Prometheus Operator upstream chart in favor of just using the CRD directory to install the CRDs. +- Added support for `rkeControllerManager`, `rkeScheduler`, `rkeProxy`, and `rkeEtcd` PushProx exporters for monitoring k8s components within RKE clusters +- Added support for a `k3sServer` PushProx exporter that monitors k3s server components (`kubeControllerManager`, `kubeScheduler`, and `kubeProxy`) within k3s clusters +- Added support for `kubeAdmControllerManager`, `kubeAdmScheduler`, `kubeAdmProxy`, and `kubeAdmEtcd` PushProx exporters for monitoring k8s components within kubeAdm clusters +- Added support for `rke2ControllerManager`, `rke2Scheduler`, `rke2Proxy`, and `rke2Etcd` PushProx exporters for monitoring k8s components within rke2 clusters +- Exposed `prometheus.prometheusSpec.ignoreNamespaceSelectors` on values.yaml and set it to `false` by default. This value instructs the default Prometheus server deployed with this chart to ignore the `namespaceSelector` field within any created ServiceMonitor or PodMonitor CRs that it selects. This prevents ServiceMonitors and PodMonitors from configuring the Prometheus scrape configuration to monitor resources outside the namespace that they are deployed in; if a user needs to have one ServiceMonitor / PodMonitor monitor resources within several namespaces (such as the resources that are used to monitor Istio in a default installation), they should not enable this option since it would require them to create one ServiceMonitor / PodMonitor CR per namespace that they would like to monitor. Relevant fields were also updated in the default README.md. +- Added `grafana.sidecar.dashboards.searchNamespace` to `values.yaml` with a default value of `cattle-dashboards`. The namespace provided should contain all ConfigMaps with the label `grafana_dashboard` and will be searched by the Grafana Dashboards sidecar for updates. The namespace specified is also created along with this deployment. All default dashboard ConfigMaps have been relocated from the deployment namespace to the namespace specified +- Added `monitoring-admin`, `monitoring-edit`, and `monitoring-view` default `ClusterRoles` to allow admins to assign roles to users to interact with Prometheus Operator CRs. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `ClusterRoleBinding` to bind these roles to a Subject to allow them to set up or view `ServiceMonitors` / `PodMonitors` / `PrometheusRules` and view `Prometheus` or `Alertmanager` CRs across the cluster. If `.Values.global.rbac.userRoles.aggregateRolesForRBAC` is enabled, these ClusterRoles will aggregate into the respective default ClusterRoles provided by Kubernetes +- Added `monitoring-config-admin`, `monitoring-config-edit` and `monitoring-config-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `Secrets` and `ConfigMaps` within the `cattle-monitoring-system` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-monitoring-system` namespace to allow them to modify Secrets / ConfigMaps tied to the deployment, such as your Alertmanager Config Secret. +- Added `monitoring-dashboard-admin`, `monitoring-dashboard-edit` and `monitoring-dashboard-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `ConfigMaps` within the `cattle-dashboards` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`) and deploying Grafana as part of this chart. In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-dashboards` namespace to allow them to create / modify ConfigMaps that contain the JSON used to persist Grafana Dashboards on the cluster. +- Added default resource limits for `Prometheus Operator`, `Prometheus`, `AlertManager`, `Grafana`, `kube-state-metrics`, `node-exporter` +- Added a default template `rancher_defaults.tmpl` to AlertManager that Rancher will offer to users in order to help configure the way alerts are rendered on a notifier. Also updated the default template deployed with this chart to reference that template and added an example of a Slack config using this template as a comment in the `values.yaml`. +- Added support for private registries via introducing a new field for `global.cattle.systemDefaultRegistry` that, if supplied, will automatically be prepended onto every image used by the chart. +- Added a default `nginx` proxy container deployed with Grafana whose config is set in the `ConfigMap` located in `charts/grafana/templates/nginx-config.yaml`. The purpose of this container is to make it possible to view Grafana's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8080` (with a `portName` of `nginx-http` instead of the default `service`), which is also where the Grafana service will now point to, and will forward all requests to the Grafana container listening on the default port `3000`. +- Added a default `nginx` proxy container deployed with Prometheus whose config is set in the `ConfigMap` located in `templates/prometheus/nginx-config.yaml`. The purpose of this container is to make it possible to view Prometheus's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8081` (with a `portName` of `nginx-http` instead of the default `web`), which is also where the Prometheus service will now point to, and will forward all requests to the Prometheus container listening on the default port `9090`. +- Added support for passing CIS Scans in a hardened cluster by introducing a Job that patches the default service account within the `cattle-monitoring-system` and `cattle-dashboards` namespaces on install or upgrade and adding a default allow all `NetworkPolicy` to the `cattle-monitoring-system` and `cattle-dashboards` namespaces. +### Modified +- Updated the chart name from `prometheus-operator` to `rancher-monitoring` and added the `io.rancher.certified: rancher` annotation to `Chart.yaml` +- Modified the default `node-exporter` port from `9100` to `9796` +- Modified the default `nameOverride` to `rancher-monitoring`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Modified the default `namespaceOverride` to `cattle-monitoring-system`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Configured some default values for `grafana.service` values and exposed them in the default README.md +- The default namespaces the following ServiceMonitors were changed from the deployment namespace to allow them to continue to monitor metrics when `prometheus.prometheusSpec.ignoreNamespaceSelectors` is enabled: + - `core-dns`: `kube-system` + - `api-server`: `default` + - `kube-controller-manager`: `kube-system` + - `kubelet`: `{{ .Values.kubelet.namespace }}` +- Disabled the following deployments by default (can be enabled if required): + - `AlertManager` + - `kube-controller-manager` metrics exporter + - `kube-etcd` metrics exporter + - `kube-scheduler` metrics exporter + - `kube-proxy` metrics exporter +- Updated default Grafana `deploymentStrategy` to `Recreate` to prevent deployments from being stuck on upgrade if a PV is attached to Grafana +- Modified the default `SelectorNilUsesHelmValues` to default to `false`. As a result, we look for all CRs with any labels in all namespaces by default rather than just the ones tagged with the label `release: rancher-monitoring`. +- Modified the default images used by the `rancher-monitoring` chart to point to Rancher mirrors of the original images from upstream. +- Modified the behavior of the chart to create the Alertmanager Config Secret via a pre-install hook instead of using the normal Helm lifecycle to manage the secret. The benefit of this approach is that all changes to the Config Secret done on a live cluster will never get overridden on a `helm upgrade` since the secret only gets created on a `helm install`. If you would like the secret to be cleaned up on an `helm uninstall`, enable `alertmanager.cleanupOnUninstall`; however, this is disabled by default to prevent the loss of alerting configuration on an uninstall. This secret will never be modified on a `helm upgrade`. +- Modified the default `securityContext` for `Pod` templates across the chart to `{"runAsNonRoot": "true", "runAsUser": "1000"}` and replaced `grafana.rbac.pspUseAppArmor` in favor of `grafana.rbac.pspAnnotations={}` in order to make it possible to deploy this chart on a hardened cluster which does not support Seccomp or AppArmor annotations in PSPs. Users can always choose to specify the annotations they want to use for the PSP directly as part of the values provided. +- Modified `.Values.prometheus.prometheusSpec.containers` to take in a string representing a template that should be rendered by Helm (via `tpl`) instead of allowing a user to provide YAML directly. +- Modified the default Grafana configuration to auto assign users who access Grafana to the Viewer role and enable anonymous access to Grafana dashboards by default. This default works well for a Rancher user who is accessing Grafana via the `kubectl proxy` on the Rancher Dashboard UI since anonymous users who enter via the proxy are authenticated by the k8s API Server, but you can / should modify this behavior if you plan on exposing Grafana in a way that does not require authentication (e.g. as a `NodePort` service). +- Modified the default Grafana configuration to add a default dashboard for Rancher on the Grafana home page. \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/CONTRIBUTING.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/CONTRIBUTING.md new file mode 100644 index 0000000000..f6ce2a3235 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contributing Guidelines + +## How to contribute to this chart + +1. Fork this repository, develop and test your Chart. +1. Bump the chart version for every change. +1. Ensure PR title has the prefix `[kube-prometheus-stack]` +1. When making changes to rules or dashboards, see the README.md section on how to sync data from upstream repositories +1. Check the `hack/minikube` folder has scripts to set up minikube and components of this chart that will allow all components to be scraped. You can use this configuration when validating your changes. +1. Check for changes of RBAC rules. +1. Check for changes in CRD specs. +1. PR must pass the linter (`helm lint`) diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/Chart.yaml new file mode 100644 index 0000000000..fa505cf7bc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/Chart.yaml @@ -0,0 +1,126 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + - name: Upstream Project + url: https://github.com/prometheus-operator/kube-prometheus + artifacthub.io/operator: "true" + catalog.cattle.io/auto-install: rancher-monitoring-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/deploys-on-os: windows + catalog.cattle.io/display-name: Monitoring + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: monitoring.coreos.com.prometheus/v1 + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: rancher-monitoring + catalog.cattle.io/requests-cpu: 4500m + catalog.cattle.io/requests-memory: 4000Mi + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: monitoring + catalog.cattle.io/upstream-version: 57.0.3 +apiVersion: v2 +appVersion: v0.72.0 +dependencies: +- condition: grafana.enabled + name: grafana + repository: file://./charts/grafana +- condition: hardenedKubelet.enabled + name: hardenedKubelet + repository: file://./charts/hardenedKubelet +- condition: hardenedNodeExporter.enabled + name: hardenedNodeExporter + repository: file://./charts/hardenedNodeExporter +- condition: k3sServer.enabled + name: k3sServer + repository: file://./charts/k3sServer +- condition: kubeStateMetrics.enabled + name: kube-state-metrics + repository: file://./charts/kube-state-metrics +- condition: kubeAdmControllerManager.enabled + name: kubeAdmControllerManager + repository: file://./charts/kubeAdmControllerManager +- condition: kubeAdmEtcd.enabled + name: kubeAdmEtcd + repository: file://./charts/kubeAdmEtcd +- condition: kubeAdmProxy.enabled + name: kubeAdmProxy + repository: file://./charts/kubeAdmProxy +- condition: kubeAdmScheduler.enabled + name: kubeAdmScheduler + repository: file://./charts/kubeAdmScheduler +- condition: prometheus-adapter.enabled + name: prometheus-adapter + repository: file://./charts/prometheus-adapter +- condition: nodeExporter.enabled + name: prometheus-node-exporter + repository: file://./charts/prometheus-node-exporter +- condition: rke2ControllerManager.enabled + name: rke2ControllerManager + repository: file://./charts/rke2ControllerManager +- condition: rke2Etcd.enabled + name: rke2Etcd + repository: file://./charts/rke2Etcd +- condition: rke2IngressNginx.enabled + name: rke2IngressNginx + repository: file://./charts/rke2IngressNginx +- condition: rke2Proxy.enabled + name: rke2Proxy + repository: file://./charts/rke2Proxy +- condition: rke2Scheduler.enabled + name: rke2Scheduler + repository: file://./charts/rke2Scheduler +- condition: rkeControllerManager.enabled + name: rkeControllerManager + repository: file://./charts/rkeControllerManager +- condition: rkeEtcd.enabled + name: rkeEtcd + repository: file://./charts/rkeEtcd +- condition: rkeIngressNginx.enabled + name: rkeIngressNginx + repository: file://./charts/rkeIngressNginx +- condition: rkeProxy.enabled + name: rkeProxy + repository: file://./charts/rkeProxy +- condition: rkeScheduler.enabled + name: rkeScheduler + repository: file://./charts/rkeScheduler +- condition: windowsExporter.enabled + name: windowsExporter + repository: file://./charts/windowsExporter +description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, + and Prometheus rules combined with documentation and scripts to provide easy to + operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus + Operator. +home: https://github.com/prometheus-operator/kube-prometheus +icon: file://assets/logos/rancher-monitoring.png +keywords: +- operator +- prometheus +- kube-prometheus +kubeVersion: '>=1.19.0-0' +maintainers: +- email: andrew@quadcorps.co.uk + name: andrewgkew +- email: gianrubio@gmail.com + name: gianrubio +- email: github.gkarthiks@gmail.com + name: gkarthiks +- email: kube-prometheus-stack@sisti.pt + name: GMartinez-Sisti +- email: github@jkroepke.de + name: jkroepke +- email: scott@r6by.com + name: scottrigby +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: quentin.bisson@gmail.com + name: QuentinBisson +name: rancher-monitoring +sources: +- https://github.com/prometheus-community/helm-charts +- https://github.com/prometheus-operator/kube-prometheus +type: application +version: 104.1.2-rc.1+up57.0.3.up57.0.3 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/README.md new file mode 100644 index 0000000000..9baf58bb16 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/README.md @@ -0,0 +1,1080 @@ +# kube-prometheus-stack + +Installs the [kube-prometheus stack](https://github.com/prometheus-operator/kube-prometheus), a collection of Kubernetes manifests, [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with [Prometheus](https://prometheus.io/) using the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). + +See the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) README for details about components, dashboards, and alerts. + +_Note: This chart was formerly named `prometheus-operator` chart, now renamed to more clearly reflect that it installs the `kube-prometheus` project stack, within which Prometheus Operator is only one component._ + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3+ + +## Get Helm Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Dependencies + +By default this chart installs additional, dependent charts: + +- [prometheus-community/kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) +- [prometheus-community/prometheus-node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) +- [grafana/grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) + +To disable dependencies during installation, see [multiple releases](#multiple-releases) below. + +_See [helm dependency](https://helm.sh/docs/helm/helm_dependency/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +CRDs created by this chart are not removed by default and should be manually cleaned up: + +```console +kubectl delete crd alertmanagerconfigs.monitoring.coreos.com +kubectl delete crd alertmanagers.monitoring.coreos.com +kubectl delete crd podmonitors.monitoring.coreos.com +kubectl delete crd probes.monitoring.coreos.com +kubectl delete crd prometheusagents.monitoring.coreos.com +kubectl delete crd prometheuses.monitoring.coreos.com +kubectl delete crd prometheusrules.monitoring.coreos.com +kubectl delete crd scrapeconfigs.monitoring.coreos.com +kubectl delete crd servicemonitors.monitoring.coreos.com +kubectl delete crd thanosrulers.monitoring.coreos.com +``` + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +With Helm v3, CRDs created by this chart are not updated by default and should be manually updated. +Consult also the [Helm Documentation on CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions). + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. + +### From 56.x to 57.x + +This version upgrades Prometheus-Operator to v0.72.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 55.x to 56.x + +This version upgrades Prometheus-Operator to v0.71.0, Prometheus to 2.49.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 54.x to 55.x + +This version upgrades Prometheus-Operator to v0.70.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 53.x to 54.x + +Grafana Helm Chart has bumped to version 7 + +Please note Grafana Helm Chart [changelog](https://github.com/grafana/helm-charts/tree/main/charts/grafana#to-700). + +### From 52.x to 53.x + +This version upgrades Prometheus-Operator to v0.69.1, Prometheus to 2.47.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 51.x to 52.x + +This includes the ability to select between using existing secrets or create new secret objects for various thanos config. The defaults have not changed but if you were setting: + +- `thanosRuler.thanosRulerSpec.alertmanagersConfig` or +- `thanosRuler.thanosRulerSpec.objectStorageConfig` or +- `thanosRuler.thanosRulerSpec.queryConfig` or +- `prometheus.prometheusSpec.thanos.objectStorageConfig` + +you will have to need to set `existingSecret` or `secret` based on your requirement + +For instance, the `thanosRuler.thanosRulerSpec.alertmanagersConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + secret: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +or the `thanosRuler.thanosRulerSpec.objectStorageConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + existingSecret: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +### From 50.x to 51.x + +This version upgrades Prometheus-Operator to v0.68.0, Prometheus to 2.47.0 and Thanos to v0.32.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 49.x to 50.x + +This version requires Kubernetes 1.19+. + +We do not expect any breaking changes in this version. + +### From 48.x to 49.x + +This version upgrades Prometheus-Operator to v0.67.1, 0, Alertmanager to v0.26.0, Prometheus to 2.46.0 and Thanos to v0.32.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 47.x to 48.x + +This version moved all CRDs into a dedicated sub-chart. No new CRDs are introduced in this version. +See [#3548](https://github.com/prometheus-community/helm-charts/issues/3548) for more context. + +We do not expect any breaking changes in this version. + +### From 46.x to 47.x + +This version upgrades Prometheus-Operator to v0.66.0 with new CRDs (PrometheusAgent and ScrapeConfig). + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 45.x to 46.x + +This version upgrades Prometheus-Operator to v0.65.1 with new CRDs (PrometheusAgent and ScrapeConfig), Prometheus to v2.44.0 and Thanos to v0.31.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 44.x to 45.x + +This version upgrades Prometheus-Operator to v0.63.0, Prometheus to v2.42.0 and Thanos to v0.30.2. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 43.x to 44.x + +This version upgrades Prometheus-Operator to v0.62.0, Prometheus to v2.41.0 and Thanos to v0.30.1. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +If you have explicitly set `prometheusOperator.admissionWebhooks.failurePolicy`, this value is now always used even when `.prometheusOperator.admissionWebhooks.patch.enabled` is `true` (the default). + +The values for `prometheusOperator.image.tag` & `prometheusOperator.prometheusConfigReloader.image.tag` are now empty by default and the Chart.yaml `appVersion` field is used instead. + +### From 42.x to 43.x + +This version upgrades Prometheus-Operator to v0.61.1, Prometheus to v2.40.5 and Thanos to v0.29.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 41.x to 42.x + +This includes the overridability of container registry for all containers at the global level using `global.imageRegistry` or per container image. The defaults have not changed but if you were using a custom image, you will have to override the registry of said custom container image before you upgrade. + +For instance, the prometheus-config-reloader used to be configured as follow: + +```yaml + image: + repository: quay.io/prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +But it now moved to: + +```yaml + image: + registry: quay.io + repository: prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +### From 40.x to 41.x + +This version upgrades Prometheus-Operator to v0.60.1, Prometheus to v2.39.1 and Thanos to v0.28.1. +This version also upgrades the Helm charts of kube-state-metrics to 4.20.2, prometheus-node-exporter to 4.3.0 and Grafana to 6.40.4. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +This version splits kubeScheduler recording and altering rules in separate config values. +Instead of `defaultRules.rules.kubeScheduler` the 2 new variables `defaultRules.rules.kubeSchedulerAlerting` and `defaultRules.rules.kubeSchedulerRecording` are used. + +### From 39.x to 40.x + +This version upgrades Prometheus-Operator to v0.59.1, Prometheus to v2.38.0, kube-state-metrics to v2.6.0 and Thanos to v0.28.0. +This version also upgrades the Helm charts of kube-state-metrics to 4.18.0 and prometheus-node-exporter to 4.2.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +Starting from prometheus-node-exporter version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i kube-prometheus-stack prometheus-community/kube-prometheus-stack +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 38.x to 39.x + +This upgraded prometheus-operator to v0.58.0 and prometheus to v2.37.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 37.x to 38.x + +Reverted one of the default metrics relabelings for cAdvisor added in 36.x, due to it breaking container_network_* and various other statistics. If you do not want this change, you will need to override the `kubelet.cAdvisorMetricRelabelings`. + +### From 36.x to 37.x + +This includes some default metric relabelings for cAdvisor and apiserver metrics to reduce cardinality. If you do not want these defaults, you will need to override the `kubeApiServer.metricRelabelings` and or `kubelet.cAdvisorMetricRelabelings`. + +### From 35.x to 36.x + +This upgraded prometheus-operator to v0.57.0 and prometheus to v2.36.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 34.x to 35.x + +This upgraded prometheus-operator to v0.56.0 and prometheus to v2.35.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 33.x to 34.x + +This upgrades to prometheus-operator to v0.55.0 and prometheus to v2.33.5. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 32.x to 33.x + +This upgrades the prometheus-node-exporter Chart to v3.0.0. Please review the changes to this subchart if you make customizations to hostMountPropagation. + +### From 31.x to 32.x + +This upgrades to prometheus-operator to v0.54.0 and prometheus to v2.33.1. It also changes the default for `grafana.serviceMonitor.enabled` to `true. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 30.x to 31.x + +This version removes the built-in grafana ServiceMonitor and instead relies on the ServiceMonitor of the sub-chart. +`grafana.serviceMonitor.enabled` must be set instead of `grafana.serviceMonitor.selfMonitor` and the old ServiceMonitor may +need to be manually cleaned up after deploying the new release. + +### From 29.x to 30.x + +This version updates kube-state-metrics to 4.3.0 and uses the new option `kube-state-metrics.releaseLabel=true` which adds the "release" label to kube-state-metrics labels, making scraping of the metrics by kube-prometheus-stack work out of the box again, independent of the used kube-prometheus-stack release name. If you already set the "release" label via `kube-state-metrics.customLabels` you might have to remove that and use it via the new option. + +### From 28.x to 29.x + +This version makes scraping port for kube-controller-manager and kube-scheduler dynamic to reflect changes to default serving ports +for those components in Kubernetes versions v1.22 and v1.23 respectively. + +If you deploy on clusters using version v1.22+, kube-controller-manager will be scraped over HTTPS on port 10257. + +If you deploy on clusters running version v1.23+, kube-scheduler will be scraped over HTTPS on port 10259. + +### From 27.x to 28.x + +This version disables PodSecurityPolicies by default because they are deprecated in Kubernetes 1.21 and will be removed in Kubernetes 1.25. + +If you are using PodSecurityPolicies you can enable the previous behaviour by setting `kube-state-metrics.podSecurityPolicy.enabled`, `prometheus-node-exporter.rbac.pspEnabled`, `grafana.rbac.pspEnabled` and `global.rbac.pspEnabled` to `true`. + +### From 26.x to 27.x + +This version splits prometheus-node-exporter chart recording and altering rules in separate config values. +Instead of `defaultRules.rules.node` the 2 new variables `defaultRules.rules.nodeExporterAlerting` and `defaultRules.rules.nodeExporterRecording` are used. + +Also the following defaultRules.rules has been removed as they had no effect: `kubeApiserverError`, `kubePrometheusNodeAlerting`, `kubernetesAbsent`, `time`. + +The ability to set a rubookUrl via `defaultRules.rules.rubookUrl` was reintroduced. + +### From 25.x to 26.x + +This version enables the prometheus-node-exporter subchart servicemonitor by default again, by setting `prometheus-node-exporter.prometheus.monitor.enabled` to `true`. + +### From 24.x to 25.x + +This version upgrade to prometheus-operator v0.53.1. It removes support for setting a runbookUrl, since the upstream format for runbooks changed. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 23.x to 24.x + +The custom `ServiceMonitor` for the _kube-state-metrics_ & _prometheus-node-exporter_ charts have been removed in favour of the built-in sub-chart `ServiceMonitor`; for both sub-charts this means that `ServiceMonitor` customisations happen via the values passed to the chart. If you haven't directly customised this behaviour then there are no changes required to upgrade, but if you have please read the following. + +For _kube-state-metrics_ the `ServiceMonitor` customisation is now set via `kube-state-metrics.prometheus.monitor` and the `kubeStateMetrics.serviceMonitor.selfMonitor.enabled` value has moved to `kube-state-metrics.selfMonitor.enabled`. + +For _prometheus-node-exporter_ the `ServiceMonitor` customisation is now set via `prometheus-node-exporter.prometheus.monitor` and the `nodeExporter.jobLabel` values has moved to `prometheus-node-exporter.prometheus.monitor.jobLabel`. + +### From 22.x to 23.x + +Port names have been renamed for Istio's +[explicit protocol selection](https://istio.io/latest/docs/ops/configuration/traffic-management/protocol-selection/#explicit-protocol-selection). + +| | old value | new value | +|-|-----------|-----------| +| `alertmanager.alertmanagerSpec.portName` | `web` | `http-web` | +| `grafana.service.portName` | `service` | `http-web` | +| `prometheus-node-exporter.service.portName` | `metrics` (hardcoded) | `http-metrics` | +| `prometheus.prometheusSpec.portName` | `web` | `http-web` | + +### From 21.x to 22.x + +Due to the upgrade of the `kube-state-metrics` chart, removal of its deployment/stateful needs to done manually prior to upgrading: + +```console +kubectl delete deployments.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +or if you use autosharding: + +```console +kubectl delete statefulsets.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +### From 20.x to 21.x + +The config reloader values have been refactored. All the values have been moved to the key `prometheusConfigReloader` and the limits and requests can now be set separately. + +### From 19.x to 20.x + +Version 20 upgrades prometheus-operator from 0.50.x to 0.52.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 18.x to 19.x + +`kubeStateMetrics.serviceMonitor.namespaceOverride` was removed. +Please use `kube-state-metrics.namespaceOverride` instead. + +### From 17.x to 18.x + +Version 18 upgrades prometheus-operator from 0.49.x to 0.50.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 16.x to 17.x + +Version 17 upgrades prometheus-operator from 0.48.x to 0.49.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 15.x to 16.x + +Version 16 upgrades kube-state-metrics to v2.0.0. This includes changed command-line arguments and removed metrics, see this [blog post](https://kubernetes.io/blog/2021/04/13/kube-state-metrics-v-2-0/). This version also removes Grafana dashboards that supported Kubernetes 1.14 or earlier. + +### From 14.x to 15.x + +Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 13.x to 14.x + +Version 14 upgrades prometheus-operator from 0.45.x to 0.46.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 12.x to 13.x + +Version 13 upgrades prometheus-operator from 0.44.x to 0.45.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +``` + +### From 11.x to 12.x + +Version 12 upgrades prometheus-operator from 0.43.x to 0.44.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.44/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +``` + +The chart was migrated to support only helm v3 and later. + +### From 10.x to 11.x + +Version 11 upgrades prometheus-operator from 0.42.x to 0.43.x. Starting with 0.43.x an additional `AlertmanagerConfigs` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.43/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +``` + +Version 11 removes the deprecated tlsProxy via ghostunnel in favor of native TLS support the prometheus-operator gained with v0.39.0. + +### From 9.x to 10.x + +Version 10 upgrades prometheus-operator from 0.38.x to 0.42.x. Starting with 0.40.x an additional `Probes` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +``` + +### From 8.x to 9.x + +Version 9 of the helm chart removes the existing `additionalScrapeConfigsExternal` in favour of `additionalScrapeConfigsSecret`. This change lets users specify the secret name and secret key to use for the additional scrape configuration of prometheus. This is useful for users that have prometheus-operator as a subchart and also have a template that creates the additional scrape configuration. + +### From 7.x to 8.x + +Due to new template functions being used in the rules in version 8.x.x of the chart, an upgrade to Prometheus Operator and Prometheus is necessary in order to support them. First, upgrade to the latest version of 7.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version 7.5.0 +``` + +Then upgrade to 8.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version [8.x.x] +``` + +Minimal recommended Prometheus version for this chart release is `2.12.x` + +### From 6.x to 7.x + +Due to a change in grafana subchart, version 7.x.x now requires Helm >= 2.12.0. + +### From 5.x to 6.x + +Due to a change in deployment labels of kube-state-metrics, the upgrade requires `helm upgrade --force` in order to re-create the deployment. If this is not done an error will occur indicating that the deployment cannot be modified: + +```console +invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/name":"kube-state-metrics"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +If this error has already been encountered, a `helm history` command can be used to determine which release has worked, then `helm rollback` to the release, then `helm upgrade --force` to this new one + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-prometheus-stack +``` + +You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. + +### Rancher Monitoring Configuration + +The following table shows values exposed by Rancher Monitoring's additions to the chart: + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `nameOverride` | Provide a name that should be used instead of the chart name when naming all resources deployed by this chart |`"rancher-monitoring"`| +| `namespaceOverride` | Override the deployment namespace | `"cattle-monitoring-system"` | +| `global.rbac.userRoles.create` | Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets | `true` | +| `global.rbac.userRoles.aggregateToDefaultRoles` | Aggregate default user ClusterRoles into default k8s ClusterRoles | `true` | +| `prometheus-adapter.enabled` | Whether to install [prometheus-adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) within the cluster | `true` | +| `prometheus-adapter.prometheus.url` | A URL pointing to the Prometheus deployment within your cluster. The default value is set based on the assumption that you plan to deploy the default Prometheus instance from this chart where `.Values.namespaceOverride=cattle-monitoring-system` and `.Values.nameOverride=rancher-monitoring` | `http://rancher-monitoring-prometheus.cattle-monitoring-system.svc` | +| `prometheus-adapter.prometheus.port` | The port on the Prometheus deployment that Prometheus Adapter can make requests to | `9090` | +| `prometheus.prometheusSpec.ignoreNamespaceSelectors` | Ignore NamespaceSelector settings from the PodMonitor and ServiceMonitor configs. If true, PodMonitors and ServiceMonitors can only discover Pods and Services within the namespace they are deployed into | `false` | + +The following values are enabled for different distributions via [rancher-pushprox](https://github.com/rancher/dev-charts/tree/master/packages/rancher-pushprox). See the rancher-pushprox `README.md` for more information on what all values can be configured for the PushProxy chart. + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `rkeControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in RKE clusters | `false` | +| `rkeScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in RKE clusters | `false` | +| `rkeProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in RKE clusters | `false` | +| `rkeIngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE clusters | `false` | +| `rkeEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in RKE clusters | `false` | +| `rke2IngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE2 clusters | `false` | +| `k3sServer.enabled` | Create a PushProx installation for monitoring k3s-server metrics (accounts for kube-controller-manager, kube-scheduler, and kube-proxy metrics) in k3s clusters | `false` | +| `kubeAdmControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in kubeAdm clusters | `false` | +| `kubeAdmScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in kubeAdm clusters | `false` | +| `kubeAdmProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in kubeAdm clusters | `false` | +| `kubeAdmEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in kubeAdm clusters | `false` | + + +### Multiple releases + +The same chart can be used to run multiple Prometheus instances in the same cluster if required. To achieve this, it is necessary to run only one instance of prometheus-operator and a pair of alertmanager pods for an HA configuration, while all other components need to be disabled. To disable a dependency during installation, set `kubeStateMetrics.enabled`, `nodeExporter.enabled` and `grafana.enabled` to `false`. + +## Work-Arounds for Known Issues + +### Running on private GKE clusters + +When Google configure the control plane for private clusters, they automatically configure VPC peering between your Kubernetes cluster’s network and a separate Google managed project. In order to restrict what Google are able to access within your cluster, the firewall rules configured restrict access to your Kubernetes pods. This means that in order to use the webhook component with a GKE private cluster, you must configure an additional firewall rule to allow the GKE control plane access to your webhook pod. + +You can read more information on how to add firewall rules for the GKE control plane nodes in the [GKE docs](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) + +Alternatively, you can disable the hooks by setting `prometheusOperator.admissionWebhooks.enabled=false`. + +## PrometheusRules Admission Webhooks + +With Prometheus Operator version 0.30+, the core Prometheus Operator pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent malformed rules from being added to the cluster. + +### How the Chart Configures the Hooks + +A validating and mutating webhook configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. + +1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. +2. The prometheus operator pod is configured to use a TLS proxy container, which will load that certificate. +3. Validating and Mutating webhook configurations are created in the cluster, with their failure mode set to Ignore. This allows rules to be created by the same chart at the same time, even though the webhook has not yet been fully set up - it does not have the correct CA field set. +4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations + +### Alternatives + +It should be possible to use [jetstack/cert-manager](https://github.com/jetstack/cert-manager) if a more complete solution is required, but it has not been tested. + +You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `prometheusOperator.admissionWebhooks.certManager.enabled` value to true. + +### Limitations + +Because the operator can only run as a single pod, there is potential for this component failure to cause rule deployment failure. Because this risk is outweighed by the benefit of having validation, the feature is enabled by default. + +## Developing Prometheus Rules and Grafana Dashboards + +This chart Grafana Dashboards and Prometheus Rules are just a copy from [prometheus-operator/prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) and other sources, synced (with alterations) by scripts in [hack](hack) folder. In order to introduce any changes you need to first [add them to the original repository](https://github.com/prometheus-operator/kube-prometheus/blob/main/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md) and then sync there by scripts. + +## Further Information + +For more in-depth documentation of configuration options meanings, please see + +- [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) +- [Prometheus](https://prometheus.io/docs/introduction/overview/) +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana#grafana-helm-chart) + +## prometheus.io/scrape + +The prometheus operator does not support annotation-based discovery of services, using the `PodMonitor` or `ServiceMonitor` CRD in its place as they provide far more configuration options. +For information on how to use PodMonitors/ServiceMonitors, please see the documentation on the `prometheus-operator/prometheus-operator` documentation here: + +- [ServiceMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-servicemonitors) +- [PodMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-podmonitors) +- [Running Exporters](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/running-exporters.md) + +By default, Prometheus discovers PodMonitors and ServiceMonitors within its namespace, that are labeled with the same release tag as the prometheus-operator release. +Sometimes, you may need to discover custom PodMonitors/ServiceMonitors, for example used to scrape data from third-party applications. +An easy way of doing this, without compromising the default PodMonitors/ServiceMonitors discovery, is allowing Prometheus to discover all PodMonitors/ServiceMonitors within its namespace, without applying label filtering. +To do so, you can set `prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues` and `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues` to `false`. + +## Migrating from stable/prometheus-operator chart + +## Zero downtime + +Since `kube-prometheus-stack` is fully compatible with the `stable/prometheus-operator` chart, a migration without downtime can be achieved. +However, the old name prefix needs to be kept. If you want the new name please follow the step by step guide below (with downtime). + +You can override the name to achieve this: + +```console +helm upgrade prometheus-operator prometheus-community/kube-prometheus-stack -n monitoring --reuse-values --set nameOverride=prometheus-operator +``` + +**Note**: It is recommended to run this first with `--dry-run --debug`. + +## Redeploy with new name (downtime) + +If the **prometheus-operator** values are compatible with the new **kube-prometheus-stack** chart, please follow the below steps for migration: + +> The guide presumes that chart is deployed in `monitoring` namespace and the deployments are running there. If in other namespace, please replace the `monitoring` to the deployed namespace. + +1. Patch the PersistenceVolume created/used by the prometheus-operator chart to `Retain` claim policy: + + ```console + kubectl patch pv/ -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + ``` + + **Note:** To execute the above command, the user must have a cluster wide permission. Please refer [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +2. Uninstall the **prometheus-operator** release and delete the existing PersistentVolumeClaim, and verify PV become Released. + + ```console + helm uninstall prometheus-operator -n monitoring + kubectl delete pvc/ -n monitoring + ``` + + Additionally, you have to manually remove the remaining `prometheus-operator-kubelet` service. + + ```console + kubectl delete service/prometheus-operator-kubelet -n kube-system + ``` + + You can choose to remove all your existing CRDs (ServiceMonitors, Podmonitors, etc.) if you want to. + +3. Remove current `spec.claimRef` values to change the PV's status from Released to Available. + + ```console + kubectl patch pv/ --type json -p='[{"op": "remove", "path": "/spec/claimRef"}]' -n monitoring + ``` + +**Note:** To execute the above command, the user must have a cluster wide permission. Please refer to [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +After these steps, proceed to a fresh **kube-prometheus-stack** installation and make sure the current release of **kube-prometheus-stack** matching the `volumeClaimTemplate` values in the `values.yaml`. + +The binding is done via matching a specific amount of storage requested and with certain access modes. + +For example, if you had storage specified as this with **prometheus-operator**: + +```yaml +volumeClaimTemplate: + spec: + storageClassName: gp2 + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi +``` + +You have to specify matching `volumeClaimTemplate` with 50Gi storage and `ReadWriteOnce` access mode. + +Additionally, you should check the current AZ of your legacy installation's PV, and configure the fresh release to use the same AZ as the old one. If the pods are in a different AZ than the PV, the release will fail to bind the existing one, hence creating a new PV. + +This can be achieved either by specifying the labels through `values.yaml`, e.g. setting `prometheus.prometheusSpec.nodeSelector` to: + +```yaml +nodeSelector: + failure-domain.beta.kubernetes.io/zone: east-west-1a +``` + +or passing these values as `--set` overrides during installation. + +The new release should now re-attach your previously released PV with its content. + +## Migrating from coreos/prometheus-operator chart + +The multiple charts have been combined into a single chart that installs prometheus operator, prometheus, alertmanager, grafana as well as the multitude of exporters necessary to monitor a cluster. + +There is no simple and direct migration path between the charts as the changes are extensive and intended to make the chart easier to support. + +The capabilities of the old chart are all available in the new chart, including the ability to run multiple prometheus instances on a single cluster - you will need to disable the parts of the chart you do not wish to deploy. + +You can check out the tickets for this change [here](https://github.com/prometheus-operator/prometheus-operator/issues/592) and [here](https://github.com/helm/charts/pull/6765). + +### High-level overview of Changes + +#### Added dependencies + +The chart has added 3 [dependencies](#dependencies). + +- Node-Exporter, Kube-State-Metrics: These components are loaded as dependencies into the chart, and are relatively simple components +- Grafana: The Grafana chart is more feature-rich than this chart - it contains a sidecar that is able to load data sources and dashboards from configmaps deployed into the same cluster. For more information check out the [documentation for the chart](https://github.com/grafana/helm-charts/blob/main/charts/grafana/README.md) + +#### Kubelet Service + +Because the kubelet service has a new name in the chart, make sure to clean up the old kubelet service in the `kube-system` namespace to prevent counting container metrics twice. + +#### Persistent Volumes + +If you would like to keep the data of the current persistent volumes, it should be possible to attach existing volumes to new PVCs and PVs that are created using the conventions in the new chart. For example, in order to use an existing Azure disk for a helm release called `prometheus-migration` the following resources can be created: + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc-prometheus-migration-prometheus-0 +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: pvc-prometheus-migration-prometheus-0 + diskURI: /subscriptions/f5125d82-2622-4c50-8d25-3f7ba3e9ac4b/resourceGroups/sample-migration-resource-group/providers/Microsoft.Compute/disks/pvc-prometheus-migration-prometheus-0 + fsType: "" + kind: Managed + readOnly: false + capacity: + storage: 1Gi + persistentVolumeReclaimPolicy: Delete + storageClassName: prometheus + volumeMode: Filesystem +``` + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: prometheus + prometheus: prometheus-migration-prometheus + name: prometheus-prometheus-migration-prometheus-db-prometheus-prometheus-migration-prometheus-0 + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: prometheus + volumeMode: Filesystem + volumeName: pvc-prometheus-migration-prometheus-0 +``` + +The PVC will take ownership of the PV and when you create a release using a persistent volume claim template it will use the existing PVCs as they match the naming convention used by the chart. For other cloud providers similar approaches can be used. + +#### KubeProxy + +The metrics bind address of kube-proxy is default to `127.0.0.1:10249` that prometheus instances **cannot** access to. You should expose metrics by changing `metricsBindAddress` field value to `0.0.0.0:10249` if you want to collect them. + +Depending on the cluster, the relevant part `config.conf` will be in ConfigMap `kube-system/kube-proxy` or `kube-system/kube-proxy-config`. For example: + +```console +kubectl -n kube-system edit cm kube-proxy +``` + +```yaml +apiVersion: v1 +data: + config.conf: |- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + # ... + # metricsBindAddress: 127.0.0.1:10249 + metricsBindAddress: 0.0.0.0:10249 + # ... + kubeconfig.conf: |- + # ... +kind: ConfigMap +metadata: + labels: + app: kube-proxy + name: kube-proxy + namespace: kube-system +``` diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/app-README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/app-README.md new file mode 100644 index 0000000000..3920854384 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/app-README.md @@ -0,0 +1,46 @@ +# Rancher Monitoring and Alerting + + This chart is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) chart. The chart deploys [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) and its CRDs along with [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana), [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) and additional charts / Kubernetes manifests to gather metrics. It allows users to monitor their Kubernetes clusters, view metrics in Grafana dashboards, and set up alerts and notifications. + +For more information on how to use the feature, refer to our [docs](https://rancher.com/docs/rancher/v2.x/en/monitoring-alerting/v2.5/). + +The chart installs the following components: + +- [Prometheus Operator](https://github.com/coreos/prometheus-operator) - The operator provides easy monitoring definitions for Kubernetes services, manages [Prometheus](https://prometheus.io/) and [AlertManager](https://prometheus.io/docs/alerting/latest/alertmanager/) instances, and adds default scrape targets for some Kubernetes components. +- [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) - A collection of community-curated Kubernetes manifests, Grafana Dashboards, and PrometheusRules that deploy a default end-to-end cluster monitoring configuration. +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) - Grafana allows a user to create / view dashboards based on the cluster metrics collected by Prometheus. +- [node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) / [kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) / [rancher-pushprox](https://github.com/rancher/charts/tree/dev-v2.7/packages/rancher-monitoring/rancher-pushprox/charts) - These charts monitor various Kubernetes components across different Kubernetes cluster types. +- [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) - The adapter allows a user to expose custom metrics, resource metrics, and external metrics on the default [Prometheus](https://prometheus.io/) instance to the Kubernetes API Server. + +For more information, review the Helm README of this chart. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Upgrading from 100.0.0+up16.6.0 to 100.1.0+up19.0.3 + +### Noticeable changes: +Grafana: +- `sidecar.dashboards.searchNamespace`, `sidecar.datasources.searchNamespace` and `sidecar.notifiers.searchNamespace` support a list of namespaces now. + +Kube-state-metrics +- the type of `collectors` is changed from Dictionary to List. +- `kubeStateMetrics.serviceMonitor.namespaceOverride` was replaced by `kube-state-metrics.namespaceOverride`. + +### Known issues: +- Occasionally, the upgrade fails with errors related to the webhook `prometheusrulemutate.monitoring.coreos.com`. This is a known issue in the upstream, and the workaround is to trigger the upgrade one more time. [32416](https://github.com/rancher/rancher/issues/32416#issuecomment-828881726) diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/.helmignore new file mode 100644 index 0000000000..8cade1318f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.vscode +.project +.idea/ +*.tmproj +OWNERS diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/Chart.yaml new file mode 100644 index 0000000000..ff6bcb26aa --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/Chart.yaml @@ -0,0 +1,39 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/grafana/helm-charts + - name: Upstream Project + url: https://github.com/grafana/grafana + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-grafana +apiVersion: v2 +appVersion: 10.4.1 +description: The leading tool for querying and visualizing time series and metrics. +home: https://grafana.com +icon: https://artifacthub.io/image/b4fed1a7-6c8f-4945-b99d-096efa3e4116 +keywords: +- monitoring +- metric +kubeVersion: '>=1.26.0-0' +maintainers: +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rluckie@cisco.com + name: rtluckie +- email: maor.friedman@redhat.com + name: maorfr +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: mail@torstenwalter.de + name: torstenwalter +name: grafana +sources: +- https://github.com/grafana/grafana +- https://github.com/grafana/helm-charts +type: application +version: 7.3.11 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/README.md new file mode 100644 index 0000000000..0ff07f297d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/README.md @@ -0,0 +1,770 @@ +# Grafana Helm Chart + +* Installs the web dashboarding system [Grafana](http://grafana.org/) + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 4.0.0 (And 3.12.1) + +This version requires Helm >= 2.12.0. + +### To 5.0.0 + +You have to add --force to your helm upgrade command as the labels of the chart have changed. + +### To 6.0.0 + +This version requires Helm >= 3.1.0. + +### To 7.0.0 + +For consistency with other Helm charts, the `global.image.registry` parameter was renamed +to `global.imageRegistry`. If you were not previously setting `global.image.registry`, no action +is required on upgrade. If you were previously setting `global.image.registry`, you will +need to instead set `global.imageRegistry`. + +## Configuration + +| Parameter | Description | Default | +|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------| +| `replicas` | Number of nodes | `1` | +| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` | +| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` | +| `podDisruptionBudget.apiVersion` | Pod disruption apiVersion | `nil` | +| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` | +| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` | +| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`| +| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `image.registry` | Image registry | `docker.io` | +| `image.repository` | Image repository | `grafana/grafana` | +| `image.tag` | Overrides the Grafana image tag whose default is the chart appVersion (`Must be >= 5.0.0`) | `` | +| `image.sha` | Image sha (optional) | `` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets (can be templated) | `[]` | +| `service.enabled` | Enable grafana service | `true` | +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Kubernetes port where service is exposed | `80` | +| `service.portName` | Name of the port on the service | `service` | +| `service.appProtocol` | Adds the appProtocol field to the service | `` | +| `service.targetPort` | Internal service is port | `3000` | +| `service.nodePort` | Kubernetes service nodePort | `nil` | +| `service.annotations` | Service annotations (can be templated) | `{}` | +| `service.labels` | Custom labels | `{}` | +| `service.clusterIP` | internal cluster service IP | `nil` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | +| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | +| `service.externalIPs` | service external IP addresses | `[]` | +| `service.externalTrafficPolicy` | change the default externalTrafficPolicy | `nil` | +| `headlessService` | Create a headless service | `false` | +| `extraExposePorts` | Additional service ports for sidecar containers| `[]` | +| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.annotations` | Ingress annotations (values are templated) | `{}` | +| `ingress.labels` | Custom labels | `{}` | +| `ingress.path` | Ingress accepted path | `/` | +| `ingress.pathType` | Ingress type of path | `Prefix` | +| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.6/guide/ingress/annotations/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| `ingress.ingressClassName` | Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 | `""` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `extraInitContainers` | Init containers to add to the grafana pod | `{}` | +| `extraContainers` | Sidecar containers to add to the grafana pod | `""` | +| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` | +| `extraLabels` | Custom labels for all manifests | `{}` | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` | +| `persistence.size` | Size of persistent volume claim | `10Gi` | +| `persistence.existingClaim` | Use an existing PVC to persist data (can be templated) | `nil` | +| `persistence.storageClassName` | Type of persistent volume claim | `nil` | +| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` | +| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | +| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | +| `persistence.extraPvcLabels` | Extra labels to apply to a PVC. | `{}` | +| `persistence.subPath` | Mount a sub dir of the persistent volume (can be templated) | `nil` | +| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` | +| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` | +| `initChownData.enabled` | If false, don't reset data ownership at startup | true | +| `initChownData.image.registry` | init-chown-data container image registry | `docker.io` | +| `initChownData.image.repository` | init-chown-data container image repository | `busybox` | +| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | +| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` | +| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` | +| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` | +| `schedulerName` | Alternate scheduler name | `nil` | +| `env` | Extra environment variables passed to pods | `{}` | +| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `envFromSecrets` | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envFromConfigMaps` | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret. (passed through [tpl](https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function)) | `{}` | +| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` | +| `extraSecretMounts` | Additional grafana server secret mounts | `[]` | +| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | +| `extraVolumes` | Additional Grafana server volumes | `[]` | +| `automountServiceAccountToken` | Mounted the service account token on the grafana pod. Mandatory, if sidecars are enabled | `true` | +| `createConfigmap` | Enable creating the grafana configmap | `true` | +| `extraConfigmapMounts` | Additional grafana server configMap volume mounts (values are templated) | `[]` | +| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | +| `plugins` | Plugins to be loaded along with Grafana | `[]` | +| `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `alerting` | Configure grafana alerting (passed through tpl) | `{}` | +| `notifiers` | Configure grafana notifiers | `{}` | +| `dashboardProviders` | Configure grafana dashboard providers | `{}` | +| `dashboards` | Dashboards to import | `{}` | +| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` | +| `grafana.ini` | Grafana's primary configuration | `{}` | +| `global.imageRegistry` | Global image pull registry for all images. | `null` | +| `global.imagePullSecrets` | Global image pull secrets (can be templated). Allows either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). | `[]` | +| `ldap.enabled` | Enable LDAP authentication | `false` | +| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` | +| `ldap.config` | Grafana's LDAP configuration | `""` | +| `annotations` | Deployment annotations | `{}` | +| `labels` | Deployment labels | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | `{}` | +| `podPortName` | Name of the grafana port on the pod | `grafana` | +| `lifecycleHooks` | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers) | `{}` | +| `sidecar.image.registry` | Sidecar image registry | `quay.io` | +| `sidecar.image.repository` | Sidecar image repository | `kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `1.26.0` | +| `sidecar.image.sha` | Sidecar image sha (optional) | `""` | +| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | +| `sidecar.resources` | Sidecar resources | `{}` | +| `sidecar.securityContext` | Sidecar securityContext | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` | +| `sidecar.alerts.enabled` | Enables the cluster wide search for alerts and adds/updates/deletes them in grafana |`false` | +| `sidecar.alerts.label` | Label that config maps with alerts should have to be added | `grafana_alert` | +| `sidecar.alerts.labelValue` | Label value that config maps with alerts should have to be added | `""` | +| `sidecar.alerts.searchNamespace` | Namespaces list. If specified, the sidecar will search for alerts config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.alerts.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.alerts.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.alerts.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/alerting/reload"` | +| `sidecar.alerts.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.alerts.initAlerts` | Set to true to deploy the alerts sidecar as an initContainer. This is needed if skipReload is true, to load any alerts defined at startup time. | `false` | +| `sidecar.alerts.extraMounts` | Additional alerts sidecar volume mounts. | `[]` | +| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | +| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | +| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | +| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` | +| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` | +| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` | +| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` | +| `sidecar.dashboards.provider.type` | Provider type | `file` | +| `sidecar.dashboards.provider.foldersFromFilesStructure` | Allow Grafana to replicate dashboard structure from filesystem. | `false` | +| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | +| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `""` | +| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | +| `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | +| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | +| `sidecar.dashboards.searchNamespace` | Namespaces list. If specified, the sidecar will search for dashboards config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` | +| `sidecar.dashboards.reloadURL` | Full url of dashboards configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/dashboards/reload"` | +| `sidecar.dashboards.skipReload` | Enabling this omits defining the REQ_USERNAME, REQ_PASSWORD, REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` | +| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | +| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `""` | +| `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.datasources.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` | +| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.datasources.initDatasources` | Set to true to deploy the datasource sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any datasources defined at startup time. | `false` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | +| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | +| `sidecar.notifiers.labelValue` | Label value that config maps with notifiers should have to be added | `""` | +| `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.notifiers.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.notifiers.reloadURL` | Full url of notifier configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/notifications/reload"` | +| `sidecar.notifiers.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.notifiers.initNotifiers` | Set to true to deploy the notifier sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any notifiers defined at startup time. | `false` | +| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | +| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | +| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | +| `admin.existingSecret` | The name of an existing secret containing the admin credentials (can be templated). | `""` | +| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` | +| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` | +| `serviceAccount.automountServiceAccountToken` | Automount the service account token on all pods where is service account is used | `false` | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `serviceAccount.create` | Create service account | `true` | +| `serviceAccount.labels` | ServiceAccount labels | `{}` | +| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` | +| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` | +| `rbac.create` | Create and use RBAC resources | `true` | +| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` | +| `rbac.useExistingRole` | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` | +| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `false` | +| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `false` | +| `rbac.extraRoleRules` | Additional rules to add to the Role | [] | +| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | +| `command` | Define command to be executed by grafana container at startup | `nil` | +| `args` | Define additional args if command is used | `nil` | +| `testFramework.enabled` | Whether to create test-related resources | `true` | +| `testFramework.image.registry` | `test-framework` image registry. | `docker.io` | +| `testFramework.image.repository` | `test-framework` image repository. | `bats/bats` | +| `testFramework.image.tag` | `test-framework` image tag. | `v1.4.1` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.securityContext` | `test-framework` securityContext | `{}` | +| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | +| `downloadDashboards.envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | +| `downloadDashboardsImage.registry` | Curl docker image registry | `docker.io` | +| `downloadDashboardsImage.repository` | Curl docker image repository | `curlimages/curl` | +| `downloadDashboardsImage.tag` | Curl docker image tag | `7.73.0` | +| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` | +| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` | +| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | +| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | +| `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` | +| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` | +| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | +| `serviceMonitor.relabelings` | RelabelConfigs to apply to samples before scraping. | `[]` | +| `serviceMonitor.metricRelabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | +| `revisionHistoryLimit` | Number of old ReplicaSets to retain | `10` | +| `imageRenderer.enabled` | Enable the image-renderer deployment & service | `false` | +| `imageRenderer.image.registry` | image-renderer Image registry | `docker.io` | +| `imageRenderer.image.repository` | image-renderer Image repository | `grafana/grafana-image-renderer` | +| `imageRenderer.image.tag` | image-renderer Image tag | `latest` | +| `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` | +| `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` | +| `imageRenderer.env` | extra env-vars for image-renderer | `{}` | +| `imageRenderer.envValueFrom` | Environment variables for image-renderer from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` | +| `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | +| `imageRenderer.podAnnotations ` | image-renderer image-renderer pod annotation | `{}` | +| `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | +| `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | +| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | +| `imageRenderer.service.portName` | image-renderer service port name | `http` | +| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` | +| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` | +| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` | +| `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | +| `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | +| `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | +| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | +| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | +| `imageRenderer.resources` | Set resource limits for image-renderer pods | `{}` | +| `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` | +| `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` | +| `imageRenderer.affinity` | Affinity settings for pod assignment | `{}` | +| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | +| `networkPolicy.ingress` | Enable the creation of an ingress network policy | `true` | +| `networkPolicy.egress.enabled` | Enable the creation of an egress network policy | `false` | +| `networkPolicy.egress.ports` | An array of ports to allow for the egress | `[]` | +| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` | + +### Example ingress with path + +With grafana 6.3 and above + +```yaml +grafana.ini: + server: + domain: monitoring.example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true +ingress: + enabled: true + hosts: + - "monitoring.example.com" + path: "/grafana" +``` + +### Example of extraVolumeMounts and extraVolumes + +Configure additional volumes with `extraVolumes` and volume mounts with `extraVolumeMounts`. + +Example for `extraVolumeMounts` and corresponding `extraVolumes`: + +```yaml +extraVolumeMounts: + - name: plugins + mountPath: /var/lib/grafana/plugins + subPath: configs/grafana/plugins + readOnly: false + - name: dashboards + mountPath: /var/lib/grafana/dashboards + hostPath: /usr/shared/grafana/dashboards + readOnly: false + +extraVolumes: + - name: plugins + existingClaim: existing-grafana-claim + - name: dashboards + hostPath: /usr/shared/grafana/dashboards +``` + +Volumes default to `emptyDir`. Set to `persistentVolumeClaim`, +`hostPath`, `csi`, or `configMap` for other types. For a +`persistentVolumeClaim`, specify an existing claim name with +`existingClaim`. + +## Import dashboards + +There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +dashboards: + default: + some-dashboard: + json: | + { + "annotations": + + ... + # Complete json file here + ... + + "title": "Some Dashboard", + "uid": "abcd1234", + "version": 1 + } + custom-dashboard: + # This is a path to a file inside the dashboards directory inside the chart directory + file: dashboards/custom-dashboard.json + prometheus-stats: + # Ref: https://grafana.com/dashboards/2 + gnetId: 2 + revision: 2 + datasource: Prometheus + loki-dashboard-quick-search: + gnetId: 12019 + revision: 2 + datasource: + - name: DS_PROMETHEUS + value: Prometheus + - name: DS_LOKI + value: Loki + local-dashboard: + url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json +``` + +## BASE64 dashboards + +Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit) +A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk. +If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk. + +### Gerrit use case + +Gerrit API for download files has the following schema: where {project-name} and +{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard +the url value is + +## Sidecar for dashboards + +If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with +a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported +dashboards are deleted/updated. + +A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside +one configmap is currently not properly mirrored in grafana. + +Example dashboard config: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-dashboard + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + [...] +``` + +## Sidecar for datasources + +If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the data sources in grafana can be imported. + +Should you aim for reloading datasources in Grafana each time the config is changed, set `sidecar.datasources.skipReload: false` and adjust `sidecar.datasources.reloadURL` to `http://..svc.cluster.local/api/admin/provisioning/datasources/reload`. + +Secrets are recommended over configmaps for this usecase because datasources usually contain private +data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example values to add a postgres datasource as a kubernetes secret: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-datasources + labels: + grafana_datasource: 'true' # default value for: sidecar.datasources.label +stringData: + pg-db.yaml: |- + apiVersion: 1 + datasources: + - name: My pg db datasource + type: postgres + url: my-postgresql-db:5432 + user: db-readonly-user + secureJsonData: + password: 'SUperSEcretPa$$word' + jsonData: + database: my_datase + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 1000 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false + # allow users to edit datasources from the UI. + editable: false +``` + +Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): + +```yaml +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + # name of the datasource. Required + - name: Graphite + # datasource type. Required + type: graphite + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: + # basic auth username + basicAuthUser: + # basic auth password + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false +``` + +## Sidecar for notifiers + +If the parameter `sidecar.notifiers.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.notifiers.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the notification channels in grafana can be imported. The secrets must be created before +`helm install` so that the notifiers init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because alert notification channels usually contain +private data like SMTP usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](https://grafana.com/docs/grafana/latest/administration/provisioning/#alert-notification-channels): + +```yaml +notifiers: + - name: notification-channel-1 + type: slack + uid: notifier1 + # either + org_id: 2 + # or + org_name: Main Org. + is_default: true + send_reminder: true + frequency: 1h + disable_resolve_message: false + # See `Supported Settings` section for settings supporter for each + # alert notification type. + settings: + recipient: 'XXX' + token: 'xoxb' + uploadImage: true + url: https://slack.com + +delete_notifiers: + - name: notification-channel-1 + uid: notifier1 + org_id: 2 + - name: notification-channel-2 + # default org_id: 1 +``` + +## Sidecar for alerting resources + +If the parameter `sidecar.alerts.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster (namespace defined by `sidecar.alerts.searchNamespace`) and filters out the ones with +a label as defined in `sidecar.alerts.label` (default is `grafana_alert`). The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported alerting resources are updated, however, deletions are a little more complicated (see below). + +This sidecar can be used to provision alert rules, contact points, notification policies, notification templates and mute timings as shown in [Grafana Documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/). + +To fetch the alert config which will be provisioned, use the alert provisioning API ([Grafana Documentation](https://grafana.com/docs/grafana/next/developers/http_api/alerting_provisioning/)). +You can use either JSON or YAML format. + +Example config for an alert rule: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-alert + labels: + grafana_alert: "1" +data: + k8s-alert.yml: |- + apiVersion: 1 + groups: + - orgId: 1 + name: k8s-alert + [...] +``` + +To delete provisioned alert rules is a two step process, you need to delete the configmap which defined the alert rule +and then create a configuration which deletes the alert rule. + +Example deletion configuration: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: delete-sample-grafana-alert + namespace: monitoring + labels: + grafana_alert: "1" +data: + delete-k8s-alert.yml: |- + apiVersion: 1 + deleteRules: + - orgId: 1 + uid: 16624780-6564-45dc-825c-8bded4ad92d3 +``` + +## Statically provision alerting resources +If you don't need to change alerting resources (alert rules, contact points, notification policies and notification templates) regularly you could use the `alerting` config option instead of the sidecar option above. +This will grab the alerting config and apply it statically at build time for the helm file. + +There are two methods to statically provision alerting configuration in Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +alerting: + team1-alert-rules.yaml: + file: alerting/team1/rules.yaml + team2-alert-rules.yaml: + file: alerting/team2/rules.yaml + team3-alert-rules.yaml: + file: alerting/team3/rules.yaml + notification-policies.yaml: + file: alerting/shared/notification-policies.yaml + notification-templates.yaml: + file: alerting/shared/notification-templates.yaml + contactpoints.yaml: + apiVersion: 1 + contactPoints: + - orgId: 1 + name: Slack channel + receivers: + - uid: default-receiver + type: slack + settings: + # Webhook URL to be filled in + url: "" + # We need to escape double curly braces for the tpl function. + text: '{{ `{{ template "default.message" . }}` }}' + title: '{{ `{{ template "default.title" . }}` }}' +``` + +The two possibilities for static alerting resource provisioning are: + +* Inlining the file contents as shown for contact points in the above example. +* Importing a file using a relative path starting from the chart root directory as shown for the alert rules in the above example. + +### Important notes on file provisioning + +* The format of the files is defined in the [Grafana documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/) on file provisioning. +* The chart supports importing YAML and JSON files. +* The filename must be unique, otherwise one volume mount will overwrite the other. +* In case of inlining, double curly braces that arise from the Grafana configuration format and are not intended as templates for the chart must be escaped. +* The number of total files under `alerting:` is not limited. Each file will end up as a volume mount in the corresponding provisioning folder of the deployed Grafana instance. +* The file size for each import is limited by what the function `.Files.Get` can handle, which suffices for most cases. + +## How to serve Grafana with a path prefix (/grafana) + +In order to serve Grafana with a prefix (e.g., ), add the following to your values.yaml. + +```yaml +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + + path: /grafana/?(.*) + hosts: + - k8s.example.dev + +grafana.ini: + server: + root_url: http://localhost:3000/grafana # this host can be localhost +``` + +## How to securely reference secrets in grafana.ini + +This example uses Grafana [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. + +In grafana.ini: + +```yaml +grafana.ini: + [auth.generic_oauth] + enabled = true + client_id = $__file{/etc/secrets/auth_generic_oauth/client_id} + client_secret = $__file{/etc/secrets/auth_generic_oauth/client_secret} +``` + +Existing secret, or created along with helm: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: auth-generic-oauth-secret +type: Opaque +stringData: + client_id: + client_secret: +``` + +Include in the `extraSecretMounts` configuration flag: + +```yaml +- extraSecretMounts: + - name: auth-generic-oauth-secret-mount + secretName: auth-generic-oauth-secret + defaultMode: 0440 + mountPath: /etc/secrets/auth_generic_oauth + readOnly: true +``` + +### extraSecretMounts using a Container Storage Interface (CSI) provider + +This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) + +```yaml +- extraSecretMounts: + - name: secrets-store-inline + mountPath: /run/secrets + readOnly: true + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "my-provider" + nodePublishSecretRef: + name: akv-creds +``` + +## Image Renderer Plug-In + +This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/README.md#run-in-docker) + +```yaml +imageRenderer: + enabled: true +``` + +### Image Renderer NetworkPolicy + +By default the image-renderer pods will have a network policy which only allows ingress traffic from the created grafana instance + +### High Availability for unified alerting + +If you want to run Grafana in a high availability cluster you need to enable +the headless service by setting `headlessService: true` in your `values.yaml` +file. + +As next step you have to setup the `grafana.ini` in your `values.yaml` in a way +that it will make use of the headless service to obtain all the IPs of the +cluster. You should replace ``{{ Name }}`` with the name of your helm deployment. + +```yaml +grafana.ini: + ... + unified_alerting: + enabled: true + ha_peers: {{ Name }}-headless:9094 + ha_listen_address: ${POD_IP}:9094 + ha_advertise_address: ${POD_IP}:9094 + + alerting: + enabled: false +``` diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/dashboards/custom-dashboard.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/dashboards/custom-dashboard.json new file mode 100644 index 0000000000..9e26dfeeb6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/dashboards/custom-dashboard.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/NOTES.txt new file mode 100644 index 0000000000..d86419fe23 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/NOTES.txt @@ -0,0 +1,55 @@ +1. Get your '{{ .Values.adminUser }}' user password by running: + + kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} -o jsonpath="{.data.{{ .Values.admin.passwordKey | default "admin-password" }}}" | base64 --decode ; echo + + +2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}.svc.cluster.local +{{ if .Values.ingress.enabled }} + If you bind grafana to 80, please update values in values.yaml and reinstall: + ``` + securityContext: + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + + command: + - "setcap" + - "'cap_net_bind_service=+ep'" + - "/usr/sbin/grafana-server &&" + - "sh" + - "/run.sh" + ``` + Details refer to https://grafana.com/docs/installation/configuration/#http-port. + Or grafana would always crash. + + From outside the cluster, the server URL(s) are: + {{- range .Values.ingress.hosts }} + http://{{ . }} + {{- end }} +{{- else }} + Get the Grafana URL to visit by running these commands in the same shell: + {{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "grafana.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT + {{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc --namespace {{ include "grafana.namespace" . }} -w {{ include "grafana.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ include "grafana.namespace" . }} {{ include "grafana.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + http://$SERVICE_IP:{{ .Values.service.port -}} + {{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ include "grafana.namespace" . }} -l "app.kubernetes.io/name={{ include "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ include "grafana.namespace" . }} port-forward $POD_NAME 3000 + {{- end }} +{{- end }} + +3. Login with the password from step 1 and the username: {{ .Values.adminUser }} + +{{- if not .Values.persistence.enabled }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Grafana pod is terminated. ##### +################################################################################# +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/_config.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/_config.tpl new file mode 100644 index 0000000000..19df19cd2a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/_config.tpl @@ -0,0 +1,171 @@ +{{/* + Generate config map data + */}} +{{- define "grafana.configData" -}} +{{ include "grafana.assertNoLeakedSecrets" . }} +{{- $files := .Files }} +{{- $root := . -}} +{{- with .Values.plugins }} +plugins: {{ join "," . }} +{{- end }} +grafana.ini: | +{{- range $elem, $elemVal := index .Values "grafana.ini" }} + {{- if not (kindIs "map" $elemVal) }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} +{{- end }} +{{- range $key, $value := index .Values "grafana.ini" }} + {{- if kindIs "map" $value }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- range $key, $value := .Values.datasources }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.notifiers }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.alerting }} +{{- if (hasKey $value "file") }} +{{ $key }}: +{{- toYaml ( $files.Get $value.file ) | nindent 2 }} +{{- else if (or (hasKey $value "secret") (hasKey $value "secretFile"))}} +{{/* will be stored inside secret generated by "configSecret.yaml"*/}} +{{- else }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.dashboardProviders }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} + +{{- if .Values.dashboards }} +download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} +{{ $dashboardProviders := .Values.dashboardProviders }} +{{- range $provider, $dashboards := .Values.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -skf \ + --connect-timeout 60 \ + --max-time 60 \ + {{- if not $value.b64content }} + {{- if not $value.acceptHeader }} + -H "Accept: application/json" \ + {{- else }} + -H "Accept: {{ $value.acceptHeader }}" \ + {{- end }} + {{- if $value.token }} + -H "Authorization: token {{ $value.token }}" \ + {{- end }} + {{- if $value.bearerToken }} + -H "Authorization: Bearer {{ $value.bearerToken }}" \ + {{- end }} + {{- if $value.basic }} + -H "Authorization: Basic {{ $value.basic }}" \ + {{- end }} + {{- if $value.gitlabToken }} + -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \ + {{- end }} + -H "Content-Type: application/json;charset=UTF-8" \ + {{- end }} + {{- $dpPath := "" -}} + {{- range $kd := (index $dashboardProviders "dashboardproviders.yaml").providers }} + {{- if eq $kd.name $provider }} + {{- $dpPath = $kd.options.path }} + {{- end }} + {{- end }} + {{- if $value.url }} + "{{ $value.url }}" \ + {{- else }} + "https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download" \ + {{- end }} + {{- if $value.datasource }} + {{- if kindIs "string" $value.datasource }} + | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g' \ + {{- end }} + {{- if kindIs "slice" $value.datasource }} + {{- range $value.datasource }} + | sed '/-- .* --/! s/${{"{"}}{{ .name }}}/{{ .value }}/g' \ + {{- end }} + {{- end }} + {{- end }} + {{- if $value.b64content }} + | base64 -d \ + {{- end }} + > "{{- if $dpPath -}}{{ $dpPath }}{{- else -}}/var/lib/grafana/dashboards/{{ $provider }}{{- end -}}/{{ $key }}.json" + {{ end }} + {{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Generate dashboard json config map data + */}} +{{- define "grafana.configDashboardProviderData" -}} +provider.yaml: |- + apiVersion: 1 + providers: + - name: '{{ .Values.sidecar.dashboards.provider.name }}' + orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + {{- end }} + type: {{ .Values.sidecar.dashboards.provider.type }} + disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} + allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }} + options: + foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} +{{- end -}} + +{{- define "grafana.secretsData" -}} +{{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} +admin-user: {{ .Values.adminUser | b64enc | quote }} +{{- if .Values.adminPassword }} +admin-password: {{ .Values.adminPassword | b64enc | quote }} +{{- else }} +admin-password: {{ include "grafana.password" . }} +{{- end }} +{{- end }} +{{- if not .Values.ldap.existingSecret }} +ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/_helpers.tpl new file mode 100644 index 0000000000..68d2d815d8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,305 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "grafana.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "grafana.serviceAccountNameTest" -}} +{{- if .Values.serviceAccount.create }} +{{- default (print (include "grafana.fullname" .) "-test") .Values.serviceAccount.nameTest }} +{{- else }} +{{- default "default" .Values.serviceAccount.nameTest }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "grafana.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.extraLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "grafana.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.imageRenderer.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.imageRenderer.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels ImageRenderer +*/}} +{{- define "grafana.imageRenderer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Looks if there's an existing secret and reuse its password. If not it generates +new password and use it. +*/}} +{{- define "grafana.password" -}} +{{- $secret := (lookup "v1" "Secret" (include "grafana.namespace" .) (include "grafana.fullname" .) ) }} +{{- if $secret }} +{{- index $secret "data" "admin-password" }} +{{- else }} +{{- (randAlphaNum 40) | b64enc | quote }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for rbac. +*/}} +{{- define "grafana.rbac.apiVersion" -}} +{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} +{{- print "rbac.authorization.k8s.io/v1" }} +{{- else }} +{{- print "rbac.authorization.k8s.io/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "grafana.ingress.apiVersion" -}} +{{- if and ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) }} +{{- print "networking.k8s.io/v1" }} +{{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +{{- print "networking.k8s.io/v1beta1" }} +{{- else }} +{{- print "extensions/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "grafana.hpa.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "autoscaling/v2" }} +{{- print "autoscaling/v2" }} +{{- else }} +{{- print "autoscaling/v2beta2" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "grafana.podDisruptionBudget.apiVersion" -}} +{{- if $.Values.podDisruptionBudget.apiVersion }} +{{- print $.Values.podDisruptionBudget.apiVersion }} +{{- else if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" }} +{{- print "policy/v1" }} +{{- else }} +{{- print "policy/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return if ingress is stable. +*/}} +{{- define "grafana.ingress.isStable" -}} +{{- eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1" }} +{{- end }} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "grafana.ingress.supportsIngressClassName" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "grafana.ingress.supportsPathType" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "root" . "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "grafana.imagePullSecrets" -}} +{{- $root := .root }} +{{- range (concat .root.Values.global.imagePullSecrets .imagePullSecrets) }} +{{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml (dict "name" (tpl .name $root)) | trim }} +{{- else }} +- name: {{ tpl . $root }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* + Checks whether or not the configSecret secret has to be created + */}} +{{- define "grafana.shouldCreateConfigSecret" -}} +{{- $secretFound := false -}} +{{- range $key, $value := .Values.datasources }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} + {{- if (or (hasKey $value "secret") (hasKey $value "secretFile")) }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- $secretFound}} +{{- end -}} + +{{/* + Checks whether the user is attempting to store secrets in plaintext + in the grafana.ini configmap +*/}} +{{/* grafana.assertNoLeakedSecrets checks for sensitive keys in values */}} +{{- define "grafana.assertNoLeakedSecrets" -}} + {{- $sensitiveKeysYaml := ` +sensitiveKeys: +- path: ["database", "password"] +- path: ["smtp", "password"] +- path: ["security", "secret_key"] +- path: ["security", "admin_password"] +- path: ["auth.basic", "password"] +- path: ["auth.ldap", "bind_password"] +- path: ["auth.google", "client_secret"] +- path: ["auth.github", "client_secret"] +- path: ["auth.gitlab", "client_secret"] +- path: ["auth.generic_oauth", "client_secret"] +- path: ["auth.okta", "client_secret"] +- path: ["auth.azuread", "client_secret"] +- path: ["auth.grafana_com", "client_secret"] +- path: ["auth.grafananet", "client_secret"] +- path: ["azure", "user_identity_client_secret"] +- path: ["unified_alerting", "ha_redis_password"] +- path: ["metrics", "basic_auth_password"] +- path: ["external_image_storage.s3", "secret_key"] +- path: ["external_image_storage.webdav", "password"] +- path: ["external_image_storage.azure_blob", "account_key"] +` | fromYaml -}} + {{- if $.Values.assertNoLeakedSecrets -}} + {{- $grafanaIni := index .Values "grafana.ini" -}} + {{- range $_, $secret := $sensitiveKeysYaml.sensitiveKeys -}} + {{- $currentMap := $grafanaIni -}} + {{- $shouldContinue := true -}} + {{- range $index, $elem := $secret.path -}} + {{- if and $shouldContinue (hasKey $currentMap $elem) -}} + {{- if eq (len $secret.path) (add1 $index) -}} + {{- if not (regexMatch "\\$(?:__(?:env|file|vault))?{[^}]+}" (index $currentMap $elem)) -}} + {{- fail (printf "Sensitive key '%s' should not be defined explicitly in values. Use variable expansion instead. You can disable this client-side validation by changing the value of assertNoLeakedSecrets." (join "." $secret.path)) -}} + {{- end -}} + {{- else -}} + {{- $currentMap = index $currentMap $elem -}} + {{- end -}} + {{- else -}} + {{- $shouldContinue = false -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/_pod.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/_pod.tpl new file mode 100644 index 0000000000..2ebf7d5f10 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/_pod.tpl @@ -0,0 +1,1296 @@ +{{- define "grafana.pod" -}} +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- $root := . -}} +{{- with .Values.schedulerName }} +schedulerName: "{{ . }}" +{{- end }} +serviceAccountName: {{ include "grafana.serviceAccountName" . }} +automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} +{{- with .Values.securityContext }} +securityContext: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.hostAliases }} +hostAliases: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .Values.dnsPolicy }} +dnsPolicy: {{ .Values.dnsPolicy }} +{{- end }} +{{- with .Values.dnsConfig }} +dnsConfig: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.priorityClassName }} +priorityClassName: {{ . }} +{{- end }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.extraInitContainers (and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts) (and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources) (and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers)) }} +initContainers: +{{- end }} +{{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} + - name: init-chown-data + {{- $registry := include "system_default_registry" . | default .Values.initChownData.image.registry -}} + {{- if .Values.initChownData.image.sha }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}@sha256:{{ .Values.initChownData.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.initChownData.image.pullPolicy }} + {{- with .Values.initChownData.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + command: + - chown + - -R + - {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }} + - /var/lib/grafana + {{- with .Values.initChownData.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} +{{- end }} +{{- if .Values.dashboards }} + - name: download-dashboards + {{- $registry := include "system_default_registry" . | default .Values.downloadDashboardsImage.registry -}} + {{- if .Values.downloadDashboardsImage.sha }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}@sha256:{{ .Values.downloadDashboardsImage.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} + command: ["/bin/sh"] + args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh -x /etc/grafana/download_dashboards.sh" ] + {{- with .Values.downloadDashboards.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + env: + {{- range $key, $value := .Values.downloadDashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.downloadDashboards.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- with .Values.downloadDashboards.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.downloadDashboards.envFromSecret }} + envFrom: + - secretRef: + name: {{ tpl . $root }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts }} + - name: {{ include "grafana.name" . }}-init-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources }} + - name: {{ include "grafana.name" . }}-init-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.datasources.searchNamespace | join ",") . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end }} +{{- if and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers }} + - name: {{ include "grafana.name" . }}-init-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- with .Values.extraInitContainers }} + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 2 }} +{{- end }} +{{- if not .Values.enableKubeBackwardCompatibility }} +enableServiceLinks: {{ .Values.enableServiceLinks }} +{{- end }} +containers: +{{- if and .Values.sidecar.alerts.enabled (not .Values.sidecar.alerts.initAlerts) }} + - name: {{ include "grafana.name" . }}-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.alerts.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.alerts.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.alerts.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.alerts.watchServerTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchServerTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.alerts.watchClientTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchClientTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if .Values.sidecar.dashboards.enabled }} + - name: {{ include "grafana.name" . }}-sc-dashboard + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.dashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.datasources.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.dashboards.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.dashboards.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.dashboards.label }}" + {{- with .Values.sidecar.dashboards.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.dashboards.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.folderAnnotation }} + - name: FOLDER_ANNOTATION + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- if not .Values.sidecar.dashboards.skipReload }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + - name: REQ_URL + value: {{ .Values.sidecar.dashboards.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.dashboards.watchServerTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchServerTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchClientTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchClientTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: {{ .Values.sidecar.dashboards.watchClientTimeout | quote }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- with .Values.sidecar.dashboards.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if and .Values.sidecar.datasources.enabled (not .Values.sidecar.datasources.initDatasources) }} + - name: {{ include "grafana.name" . }}-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.datasources.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.datasources.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.datasources.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.datasources.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.datasources.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.datasources.watchServerTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchServerTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.watchClientTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchClientTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: {{ include "grafana.name" . }}-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.notifiers.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.notifiers.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.notifiers.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.notifiers.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.notifiers.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.notifiers.watchServerTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchServerTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.notifiers.watchClientTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchClientTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: {{ include "grafana.name" . }}-sc-plugins + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.plugins.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.plugins.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.plugins.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.plugins.label }}" + {{- if .Values.sidecar.plugins.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.plugins.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/plugins" + - name: RESOURCE + value: {{ quote .Values.sidecar.plugins.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.plugins.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.plugins.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.plugins.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.plugins.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.plugins.watchServerTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchServerTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.watchClientTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchClientTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" +{{- end}} + - name: {{ .Chart.Name }} + {{- $registry := include "system_default_registry" . | default .Values.image.registry -}} + {{- if .Values.image.sha }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .Values.args }} + args: + {{- range .Values.args }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + {{- if .Values.ldap.enabled }} + - name: ldap + mountPath: "/etc/grafana/ldap.toml" + subPath: ldap.toml + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ tpl (.subPath | default "") $root }} + readOnly: {{ .readOnly }} + {{- end }} + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- with .Values.dashboards }} + {{- range $provider, $dashboards := . }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "json") (hasKey $value "file")) }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardsConfigMaps }} + {{- range (keys . | sortAlpha) }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" + {{- end }} + {{- end }} + {{- with .Values.datasources }} + {{- $datasources := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $datasources .) "secret")) }} {{/*check if current datasource should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.notifiers }} + {{- $notifiers := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $notifiers .) "secret")) }} {{/*check if current notifier should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.alerting }} + {{- $alertingmap := .}} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $.Values.alerting .) "secret") (hasKey (index $.Values.alerting .) "secretFile")) }} {{/*check if current alerting entry should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardProviders }} + {{- range (keys . | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- end}} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml + {{- end}} + {{- end}} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" + {{- end}} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" + {{- end}} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" + {{- end}} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + ports: + - name: {{ .Values.podPortName }} + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + - name: {{ .Values.gossipPortName }}-tcp + containerPort: 9094 + protocol: TCP + - name: {{ .Values.gossipPortName }}-udp + containerPort: 9094 + protocol: UDP + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if .Values.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ include "grafana.fullname" . }} + key: plugins + {{- end }} + {{- if .Values.smtp.existingSecret }} + - name: GF_SMTP_USER + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.userKey | default "user" }} + - name: GF_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.passwordKey | default "password" }} + {{- end }} + {{- if .Values.imageRenderer.enabled }} + - name: GF_RENDERING_SERVER_URL + value: http://{{ include "grafana.fullname" . }}-image-renderer.{{ include "grafana.namespace" . }}:{{ .Values.imageRenderer.service.port }}/render + - name: GF_RENDERING_CALLBACK_URL + value: {{ .Values.imageRenderer.grafanaProtocol }}://{{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} + {{- end }} + - name: GF_PATHS_DATA + value: {{ (get .Values "grafana.ini").paths.data }} + - name: GF_PATHS_LOGS + value: {{ (get .Values "grafana.ini").paths.logs }} + - name: GF_PATHS_PLUGINS + value: {{ (get .Values "grafana.ini").paths.plugins }} + - name: GF_PATHS_PROVISIONING + value: {{ (get .Values "grafana.ini").paths.provisioning }} + {{- range $key, $value := .Values.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- range $key, $value := .Values.env }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" + {{- end }} + {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) .Values.envFromConfigMaps }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ tpl .Values.envFromSecret . }} + {{- end }} + {{- if .Values.envRenderSecret }} + - secretRef: + name: {{ include "grafana.fullname" . }}-env + {{- end }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- range .Values.envFromConfigMaps }} + - configMapRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.lifecycleHooks }} + lifecycle: + {{- tpl (toYaml .) $root | nindent 6 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- with .Values.extraContainers }} + {{- tpl . $ | nindent 2 }} +{{- end }} +nodeSelector: {{ include "linux-node-selector" . | nindent 2 }} +{{- with .Values.nodeSelector }} + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.affinity }} +affinity: + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- with .Values.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} +tolerations: {{ include "linux-node-tolerations" . | nindent 2 }} +{{- with .Values.tolerations }} + {{- toYaml . | nindent 2 }} +{{- end }} +volumes: + - name: config + configMap: + name: {{ include "grafana.fullname" . }} + {{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} + {{- if and .Values.createConfigmap $createConfigSecret }} + - name: config-secret + secret: + secretName: {{ include "grafana.fullname" . }}-config-secret + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.dashboards }} + {{- range (keys .Values.dashboards | sortAlpha) }} + - name: dashboards-{{ . }} + configMap: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.dashboardsConfigMaps }} + {{- range $provider, $name := .Values.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ tpl $name $root }} + {{- end }} + {{- end }} + {{- if .Values.ldap.enabled }} + - name: ldap + secret: + {{- if .Values.ldap.existingSecret }} + secretName: {{ .Values.ldap.existingSecret }} + {{- else }} + secretName: {{ include "grafana.fullname" . }} + {{- end }} + items: + - key: ldap-toml + path: ldap.toml + {{- end }} + {{- if and .Values.persistence.enabled (eq .Values.persistence.type "pvc") }} + - name: storage + persistentVolumeClaim: + claimName: {{ tpl (.Values.persistence.existingClaim | default (include "grafana.fullname" .)) . }} + {{- else if and .Values.persistence.enabled (has .Values.persistence.type $sts) }} + {{/* nothing */}} + {{- else }} + - name: storage + {{- if .Values.persistence.inMemory.enabled }} + emptyDir: + medium: Memory + {{- with .Values.persistence.inMemory.sizeLimit }} + sizeLimit: {{ . }} + {{- end }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + emptyDir: + {{- with .Values.sidecar.alerts.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + emptyDir: + {{- with .Values.sidecar.dashboards.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + configMap: + name: {{ include "grafana.fullname" . }}-config-dashboards + {{- end }} + {{- end }} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + emptyDir: + {{- with .Values.sidecar.datasources.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + emptyDir: + {{- with .Values.sidecar.plugins.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + emptyDir: + {{- with .Values.sidecar.notifiers.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- range .Values.extraSecretMounts }} + {{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else if .projected }} + - name: {{ .name }} + projected: + {{- toYaml .projected | nindent 6 }} + {{- else if .csi }} + - name: {{ .name }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- end }} + {{- end }} + {{- range .Values.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 6 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 6 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 6 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + emptyDir: {} + {{- end }} + {{- with .Values.extraContainerVolumes }} + {{- tpl (toYaml .) $root | nindent 2 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/clusterrole.yaml new file mode 100644 index 0000000000..3af4b62b63 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) (not .Values.rbac.useExistingClusterRole) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-clusterrole +{{- if or .Values.sidecar.dashboards.enabled .Values.rbac.extraClusterRoleRules .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} +rules: + {{- if or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end}} + {{- with .Values.rbac.extraClusterRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end}} +{{- end}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..bda9431a2c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "grafana.fullname" . }}-clusterrolebinding + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +roleRef: + kind: ClusterRole + {{- if .Values.rbac.useExistingClusterRole }} + name: {{ .Values.rbac.useExistingClusterRole }} + {{- else }} + name: {{ include "grafana.fullname" . }}-clusterrole + {{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/configSecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/configSecret.yaml new file mode 100644 index 0000000000..55574b9bbc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/configSecret.yaml @@ -0,0 +1,43 @@ +{{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} +{{- if and .Values.createConfigmap $createConfigSecret }} +{{- $files := .Files }} +{{- $root := . -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ include "grafana.fullname" . }}-config-secret" + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- range $key, $value := .Values.alerting }} + {{- if (hasKey $value "secretFile") }} + {{- $key | nindent 2 }}: + {{- toYaml ( $files.Get $value.secretFile ) | b64enc | nindent 4}} + {{/* as of https://helm.sh/docs/chart_template_guide/accessing_files/ this will only work if you fork this chart and add files to it*/}} + {{- end }} +{{- end }} +stringData: +{{- range $key, $value := .Values.datasources }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} +{{ if (hasKey $value "secret") }} + {{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml new file mode 100644 index 0000000000..b412c4d1f0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.sidecar.dashboards.enabled .Values.sidecar.dashboards.SCProvider }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-config-dashboards + namespace: {{ include "grafana.namespace" . }} +data: + {{- include "grafana.configDashboardProviderData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/configmap.yaml new file mode 100644 index 0000000000..7d7428be51 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/configmap.yaml @@ -0,0 +1,15 @@ +{{- if .Values.createConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- include "grafana.configData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml new file mode 100644 index 0000000000..b96ce72026 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml @@ -0,0 +1,38 @@ +{{- if .Values.dashboards }} +{{ $files := .Files }} +{{- range $provider, $dashboards := .Values.dashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ include "grafana.namespace" $ }} + labels: + {{- include "grafana.labels" $ | nindent 4 }} + dashboard-provider: {{ $provider }} + {{- if $.Values.sidecar.dashboards.enabled }} + {{ $.Values.sidecar.dashboards.label }}: {{ $.Values.sidecar.dashboards.labelValue | quote }} + {{- end }} +{{- if $dashboards }} +data: +{{- $dashboardFound := false }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} +{{- $dashboardFound = true }} + {{- print $key | nindent 2 }}.json: + {{- if hasKey $value "json" }} + |- + {{- $value.json | nindent 6 }} + {{- end }} + {{- if hasKey $value "file" }} + {{- toYaml ( $files.Get $value.file ) | nindent 4}} + {{- end }} +{{- end }} +{{- end }} +{{- if not $dashboardFound }} + {} +{{- end }} +{{- end }} +--- +{{- end }} + +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/deployment.yaml new file mode 100644 index 0000000000..46c016faa3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/deployment.yaml @@ -0,0 +1,53 @@ +{{- if (and (not .Values.useStatefulSet) (or (not .Values.persistence.enabled) (eq .Values.persistence.type "pvc"))) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }} + replicas: {{ .Values.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + {{- with .Values.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include "grafana.configData" . | sha256sum }} + {{- if .Values.dashboards }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + {{- end }} + checksum/sc-dashboard-provider-config: {{ include "grafana.configDashboardProviderData" . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include "grafana.secretsData" . | sha256sum }} + {{- end }} + {{- if .Values.envRenderSecret }} + checksum/secret-env: {{ tpl (toYaml .Values.envRenderSecret) . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/extra-manifests.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/extra-manifests.yaml new file mode 100644 index 0000000000..a9bb3b6ba8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraObjects }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/headless-service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/headless-service.yaml new file mode 100644 index 0000000000..3028589d32 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/headless-service.yaml @@ -0,0 +1,22 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if or .Values.headlessService (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-headless + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + clusterIP: None + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + type: ClusterIP + ports: + - name: {{ .Values.gossipPortName }}-tcp + port: 9094 +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/hpa.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/hpa.yaml new file mode 100644 index 0000000000..46bbcb49a2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/hpa.yaml @@ -0,0 +1,52 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if .Values.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }} + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + {{- if has .Values.persistence.type $sts }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ include "grafana.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.behavior }} + behavior: {{ toYaml .Values.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml new file mode 100644 index 0000000000..28231b803e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml @@ -0,0 +1,131 @@ +{{ if .Values.imageRenderer.enabled }} +{{- $root := . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.imageRenderer.autoscaling.enabled) (.Values.imageRenderer.replicas) }} + replicas: {{ .Values.imageRenderer.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + + {{- with .Values.imageRenderer.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 8 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.imageRenderer.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imageRenderer.schedulerName }} + schedulerName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.serviceAccountName }} + serviceAccountName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.image.pullSecrets }} + imagePullSecrets: + {{- range . }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + containers: + - name: {{ .Chart.Name }}-image-renderer + {{- $registry := include "system_default_registry" | default .Values.imageRenderer.image.registry -}} + {{- if .Values.imageRenderer.image.sha }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}@sha256:{{ .Values.imageRenderer.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.imageRenderer.image.pullPolicy }} + {{- if .Values.imageRenderer.command }} + command: + {{- range .Values.imageRenderer.command }} + - {{ . }} + {{- end }} + {{- end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + containerPort: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.imageRenderer.service.portName }} + env: + - name: HTTP_PORT + value: {{ .Values.imageRenderer.service.targetPort | quote }} + {{- if .Values.imageRenderer.serviceMonitor.enabled }} + - name: ENABLE_METRICS + value: "true" + {{- end }} + {{- range $key, $value := .Values.imageRenderer.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 16 }} + {{- end }} + {{- range $key, $value := .Values.imageRenderer.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- with .Values.imageRenderer.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /tmp + name: image-renderer-tmpfs + {{- with .Values.imageRenderer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.imageRenderer.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: image-renderer-tmpfs + emptyDir: {} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml new file mode 100644 index 0000000000..b0f0059b79 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "grafana.fullname" . }}-image-renderer + minReplicas: {{ .Values.imageRenderer.autoscaling.minReplicas }} + maxReplicas: {{ .Values.imageRenderer.autoscaling.maxReplicas }} + metrics: + {{- if .Values.imageRenderer.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.behavior }} + behavior: {{ toYaml .Values.imageRenderer.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml new file mode 100644 index 0000000000..d1a0eb313d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitIngress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-ingress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer ingress traffic from grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Ingress + ingress: + - ports: + - port: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.imageRenderer.networkPolicy.extraIngressSelectors -}} + {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} + +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitEgress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-egress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer egress traffic to grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Egress + egress: + # allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # talk only to grafana + - ports: + - port: {{ .Values.service.targetPort }} + protocol: TCP + to: + - namespaceSelector: + matchLabels: + name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-service.yaml new file mode 100644 index 0000000000..f8da127cf8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-service.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + {{- with .Values.imageRenderer.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + port: {{ .Values.imageRenderer.service.port }} + protocol: TCP + targetPort: {{ .Values.imageRenderer.service.targetPort }} + {{- with .Values.imageRenderer.appProtocol }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml new file mode 100644 index 0000000000..5d9f09d266 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if .Values.imageRenderer.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + {{- if .Values.imageRenderer.serviceMonitor.namespace }} + namespace: {{ tpl .Values.imageRenderer.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.imageRenderer.service.portName }} + {{- with .Values.imageRenderer.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.imageRenderer.serviceMonitor.path }} + scheme: {{ .Values.imageRenderer.serviceMonitor.scheme }} + {{- with .Values.imageRenderer.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}-image-renderer" + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.imageRenderer.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/ingress.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/ingress.yaml new file mode 100644 index 0000000000..b2ffd81095 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/ingress.yaml @@ -0,0 +1,78 @@ +{{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "grafana.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "grafana.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "grafana.ingress.supportsPathType" .) "true" -}} +{{- $fullName := include "grafana.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +{{- $extraPaths := .Values.ingress.extraPaths -}} +apiVersion: {{ include "grafana.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} + {{- with .Values.ingress.tls }} + tls: + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} + rules: + {{- if .Values.ingress.hosts }} + {{- range .Values.ingress.hosts }} + - host: {{ tpl . $ | quote }} + http: + paths: + {{- with $extraPaths }} + {{- toYaml . | nindent 10 }} + {{- end }} + - path: {{ $ingressPath }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end }} + {{- else }} + - http: + paths: + - backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- with $ingressPath }} + path: {{ . }} + {{- end }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/networkpolicy.yaml new file mode 100644 index 0000000000..4cd3ed6976 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/networkpolicy.yaml @@ -0,0 +1,61 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + policyTypes: + {{- if .Values.networkPolicy.ingress }} + - Ingress + {{- end }} + {{- if .Values.networkPolicy.egress.enabled }} + - Egress + {{- end }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + + {{- if .Values.networkPolicy.egress.enabled }} + egress: + {{- if not .Values.networkPolicy.egress.blockDNSResolution }} + - ports: + - port: 53 + protocol: UDP + {{- end }} + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- with .Values.networkPolicy.egress.to }} + to: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.networkPolicy.ingress }} + ingress: + - ports: + - port: {{ .Values.service.targetPort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ include "grafana.fullname" . }}-client: "true" + {{- with .Values.networkPolicy.explicitNamespacesSelector }} + - namespaceSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "grafana.labels" . | nindent 14 }} + role: read + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/nginx-config.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/nginx-config.yaml new file mode 100644 index 0000000000..557471f6ff --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/nginx-config.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-nginx-proxy-config + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 8080; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location /api/dashboards { + proxy_pass http://localhost:3000; + } + + location /api/search { + proxy_pass http://localhost:3000; + + sub_filter_types application/json; + sub_filter_once off; + } + + location /api/live/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $http_host; + proxy_pass http://localhost:3000; + } + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:3000/; + + sub_filter_once off; + + {{- if eq .Values.global.cattle.clusterId "local" -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- else -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- end -}} + + sub_filter ':"/avatar/' ':"avatar/'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000000..05251214ac --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: {{ include "grafana.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..973caccd57 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml @@ -0,0 +1,45 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + # Default set from Docker, with DAC_OVERRIDE and CHOWN + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'csi' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/pvc.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/pvc.yaml new file mode 100644 index 0000000000..c9b234305f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/pvc.yaml @@ -0,0 +1,41 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.persistence.extraPvcLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.finalizers }} + finalizers: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)) }} + volumeName: {{ (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)).spec.volumeName }} + {{- end }} + {{- with .Values.persistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/role.yaml new file mode 100644 index 0000000000..469b6f4e6c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/role.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)) }} +rules: + {{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}] + {{- end }} + {{- if and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled) }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- with .Values.rbac.extraRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/rolebinding.yaml new file mode 100644 index 0000000000..58f77c6b0b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/rolebinding.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + {{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} + {{- else }} + name: {{ include "grafana.fullname" . }} + {{- end }} +subjects: +- kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/secret-env.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/secret-env.yaml new file mode 100644 index 0000000000..eb14aac707 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/secret-env.yaml @@ -0,0 +1,14 @@ +{{- if .Values.envRenderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }}-env + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $key, $val := .Values.envRenderSecret }} + {{ $key }}: {{ tpl ($val | toString) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/secret.yaml new file mode 100644 index 0000000000..fd2ca50f4b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/secret.yaml @@ -0,0 +1,16 @@ +{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- include "grafana.secretsData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/service.yaml new file mode 100644 index 0000000000..e9396a15c6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/service.yaml @@ -0,0 +1,61 @@ +{{- if .Values.service.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} +spec: + {{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- with .Values.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- else if eq .Values.service.type "LoadBalancer" }} + type: LoadBalancer + {{- with .Values.service.loadBalancerIP }} + loadBalancerIP: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerClass }} + loadBalancerClass: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} + type: {{ .Values.service.type }} + {{- end }} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} + {{- with .Values.service.appProtocol }} + appProtocol: {{ . }} + {{- end }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- with .Values.extraExposePorts }} + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/serviceaccount.yaml new file mode 100644 index 0000000000..ffca0717ae --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.serviceAccount.autoMount | default .Values.serviceAccount.automountServiceAccountToken }} +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/servicemonitor.yaml new file mode 100644 index 0000000000..b321b1269c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/servicemonitor.yaml @@ -0,0 +1,68 @@ +{{- if .Values.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ tpl .Values.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- with .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + metricRelabelings: + {{- if .Values.serviceMonitor.metricRelabelings }} + {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.serviceMonitor.relabelings }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + {{- with .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}" + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/statefulset.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/statefulset.yaml new file mode 100644 index 0000000000..49278083e8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/statefulset.yaml @@ -0,0 +1,58 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if (or (.Values.useStatefulSet) (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)))}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + serviceName: {{ include "grafana.fullname" . }}-headless + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} + {{- if .Values.persistence.enabled}} + volumeClaimTemplates: + - metadata: + name: storage + spec: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + accessModes: {{ .Values.persistence.accessModes }} + storageClassName: {{ .Values.persistence.storageClassName }} + resources: + requests: + storage: {{ required "Must provide size for persistent volumes used by Grafana" .Values.persistence.size }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-configmap.yaml new file mode 100644 index 0000000000..01c96c9243 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + run.sh: |- + @test "Test Health" { + url="http://{{ include "grafana.fullname" . }}/api/health" + + code=$(wget --server-response --spider --timeout 90 --tries 10 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml new file mode 100644 index 0000000000..70a0a884c9 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -0,0 +1,32 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }}-test + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +spec: + allowPrivilegeEscalation: true + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + fsGroup: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - projected + - csi + - secret +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-role.yaml new file mode 100644 index 0000000000..976418b137 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-role.yaml @@ -0,0 +1,17 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}-test] +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml new file mode 100644 index 0000000000..509566eccd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "grafana.fullname" . }}-test +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml new file mode 100644 index 0000000000..38fba3596a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.testFramework.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test.yaml new file mode 100644 index 0000000000..83aaa185c2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/templates/tests/test.yaml @@ -0,0 +1,53 @@ +{{- if .Values.testFramework.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + namespace: {{ include "grafana.namespace" . }} +spec: + serviceAccountName: {{ include "grafana.serviceAccountNameTest" . }} + {{- with .Values.testFramework.securityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 4 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 4 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: {{ .Release.Name }}-test + image: "{{ template "system_default_registry" . | default .Values.testFramework.image.registry }}/{{ .Values.testFramework.image.repository }}:{{ .Values.testFramework.image.tag }}" + imagePullPolicy: "{{ .Values.testFramework.imagePullPolicy}}" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + {{- with .Values.testFramework.resources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tests + configMap: + name: {{ include "grafana.fullname" . }}-test + restartPolicy: Never +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/values.yaml new file mode 100644 index 0000000000..2bb62aec39 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/grafana/values.yaml @@ -0,0 +1,1315 @@ +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # Can be tempalted. + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + +rbac: + create: true + ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true) + # useExistingRole: name-of-some-role + # useExistingClusterRole: name-of-some-clusterRole + pspEnabled: false + pspUseAppArmor: false + namespaced: false + extraRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] + extraClusterRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] +serviceAccount: + create: true + name: + nameTest: + ## ServiceAccount labels. + labels: {} + ## Service account annotations. Can be templated. + # annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::123456789000:role/iam-role-name-here + + ## autoMount is deprecated in favor of automountServiceAccountToken + # autoMount: false + automountServiceAccountToken: true + +replicas: 1 + +## Create a headless service for the deployment +headlessService: false + +## Should the service account be auto mounted on the pod +automountServiceAccountToken: true + +## Create HorizontalPodAutoscaler object for deployment type +# +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + +## See `kubectl explain poddisruptionbudget.spec` for more +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} +# apiVersion: "" +# minAvailable: 1 +# maxUnavailable: 1 + +## See `kubectl explain deployment.spec.strategy` for more +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: RollingUpdate + +readinessProbe: + httpGet: + path: /api/health + port: 3000 + +livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: "default-scheduler" + +image: + repository: rancher/mirrored-grafana-grafana + # Overrides the Grafana image tag whose default is the chart appVersion + tag: 10.3.3 + sha: "" + pullPolicy: IfNotPresent + + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Can be templated. + ## + pullSecrets: [] + # - myRegistrKeySecretName + +testFramework: + enabled: false + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# dns configuration for pod +dnsPolicy: ~ +dnsConfig: {} + # nameservers: + # - 8.8.8.8 + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# Enable creating the grafana configmap +createConfigmap: true + +# Extra configmaps to mount in grafana pods +# Values are templated. +extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + + +extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + +# Apply extra labels to common labels. +extraLabels: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +downloadDashboardsImage: + repository: rancher/mirrored-curlimages-curl + tag: 7.85.0 + sha: "" + pullPolicy: IfNotPresent + +downloadDashboards: + env: {} + envFromSecret: "" + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## Pod Annotations +# podAnnotations: {} + +## Pod Labels +# podLabels: {} + +podPortName: grafana +gossipPortName: gossip +## Deployment annotations +# annotations: {} + +## Expose the grafana service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + enabled: true + type: ClusterIP + loadBalancerIP: "" + loadBalancerClass: "" + loadBalancerSourceRanges: [] + port: 80 + targetPort: 3000 + # targetPort: 4181 To be used with a proxy extraContainer + ## Service annotations. Can be templated. + annotations: {} + labels: {} + portName: service + # Adds the appProtocol field to the service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + +serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 30s + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + metricRelabelings: [] + targetLabels: [] + +extraExposePorts: [] + # - name: keycloak + # port: 8080 + # targetPort: 8080 + +# overrides pod.spec.hostAliases in the grafana deployment's pods +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "my.host.com" + +ingress: + enabled: false + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + # Values can be templated + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + path: / + + # pathType is only for k8s >= 1.1= + pathType: Prefix + + hosts: + - chart-example.local + ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## Or for k8s > 1.19 + # - path: /* + # pathType: Prefix + # backend: + # service: + # name: ssl-redirect + # port: + # name: use-annotation + + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Topology Spread Constraints +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## +topologySpreadConstraints: [] + +## Additional init containers (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +## +extraInitContainers: [] + +## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod +extraContainers: "" +# extraContainers: | +# - name: proxy +# image: quay.io/gambol99/keycloak-proxy:latest +# args: +# - -provider=github +# - -client-id= +# - -client-secret= +# - -github-org= +# - -email-domain=* +# - -cookie-secret= +# - -http-address=http://0.0.0.0:4181 +# - -upstream-url=http://127.0.0.1:3000 +# ports: +# - name: proxy-web +# containerPort: 4181 + +## Volumes that can be used in init containers that will not be mounted to deployment pods +extraContainerVolumes: [] +# - name: volume-from-secret +# secret: +# secretName: secret-to-mount +# - name: empty-dir-volume +# emptyDir: {} + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + type: pvc + enabled: false + # storageClassName: default + accessModes: + - ReadWriteOnce + size: 10Gi + # annotations: {} + finalizers: + - kubernetes.io/pvc-protection + # selectorLabels: {} + ## Sub-directory of the PV to mount. Can be templated. + # subPath: "" + ## Name of an existing PVC. Can be templated. + # existingClaim: + ## Extra labels to apply to a PVC. + extraPvcLabels: {} + + ## If persistence is not enabled, this allows to mount the + ## local storage in-memory to improve performance + ## + inMemory: + enabled: false + ## The maximum usage on memory medium EmptyDir would be + ## the minimum value between the SizeLimit specified + ## here and the sum of memory limits of all containers in a pod + ## + # sizeLimit: 300Mi + +initChownData: + ## If false, data ownership will not be reset at startup + ## This allows the grafana-server to be run with an arbitrary user + ## + enabled: true + + ## initChownData container image + ## + image: + repository: rancher/mirrored-library-busybox + tag: "1.31.1" + sha: "" + pullPolicy: IfNotPresent + + ## initChownData resource requests and limits + ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + securityContext: + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + capabilities: + add: + - CHOWN + +# Administrator credentials when not using an existing secret (see below) +adminUser: admin +# adminPassword: strongpassword + +# Use an existing secret for the admin user. +admin: + ## Name of the secret. Can be templated. + existingSecret: "" + userKey: admin-user + passwordKey: admin-password + +## Define command to be executed at startup by grafana container +## Needed if using `vault-env` to manage secrets (ref: https://banzaicloud.com/blog/inject-secrets-into-pods-vault/) +## Default is "run.sh" as defined in grafana's Dockerfile +# command: +# - "sh" +# - "/run.sh" + +## Optionally define args if command is used +## Needed if using `hashicorp/envconsul` to manage secrets +## By default no arguments are set +# args: +# - "-secret" +# - "secret/grafana" +# - "./grafana" + +## Extra environment variables that will be pass onto deployment pods +## +## to provide grafana with access to CloudWatch on AWS EKS: +## 1. create an iam role of type "Web identity" with provider oidc.eks.* (note the provider for later) +## 2. edit the "Trust relationships" of the role, add a line inside the StringEquals clause using the +## same oidc eks provider as noted before (same as the existing line) +## also, replace NAMESPACE and prometheus-operator-grafana with the service account namespace and name +## +## "oidc.eks.us-east-1.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:sub": "system:serviceaccount:NAMESPACE:prometheus-operator-grafana", +## +## 3. attach a policy to the role, you can use a built in policy called CloudWatchReadOnlyAccess +## 4. use the following env: (replace 123456789000 and iam-role-name-here with your aws account number and role name) +## +## env: +## AWS_ROLE_ARN: arn:aws:iam::123456789000:role/iam-role-name-here +## AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +## AWS_REGION: us-east-1 +## +## 5. uncomment the EKS section in extraSecretMounts: below +## 6. uncomment the annotation section in the serviceAccount: above +## make sure to replace arn:aws:iam::123456789000:role/iam-role-name-here with your role arn + +env: {} + +## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core +## Renders in container spec as: +## env: +## ... +## - name: +## valueFrom: +## +envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## The name of a secret in the same kubernetes namespace which contain values to be added to the environment +## This can be useful for auth tokens, etc. Value is templated. +envFromSecret: "" + +## Sensible environment variables that will be rendered as new secret object +## This can be useful for auth tokens, etc. +## If the secret values contains "{{", they'll need to be properly escaped so that they are not interpreted by Helm +## ref: https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function +envRenderSecret: {} + +## The names of secrets in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the secret must be defined with an optional key. +## Name is templated. +envFromSecrets: [] +## - name: secret-name +## prefix: prefix +## optional: true + +## The names of conifgmaps in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the configmap must be defined with an optional key. +## Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#configmapenvsource-v1-core +envFromConfigMaps: [] +## - name: configmap-name +## prefix: prefix +## optional: true + +# Inject Kubernetes services as environment variables. +# See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables +enableServiceLinks: true + +## Additional grafana server secret mounts +# Defines additional mounts with secrets. Secrets must be manually created in the namespace. +extraSecretMounts: [] + # - name: secret-files + # mountPath: /etc/secrets + # secretName: grafana-secret-files + # readOnly: true + # subPath: "" + # + # for AWS EKS (cloudwatch) use the following (see also instruction in env: above) + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # projected: + # defaultMode: 420 + # sources: + # - serviceAccountToken: + # audience: sts.amazonaws.com + # expirationSeconds: 86400 + # path: token + # + # for CSI e.g. Azure Key Vault use the following + # - name: secrets-store-inline + # mountPath: /run/secrets + # readOnly: true + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "akv-grafana-spc" + # nodePublishSecretRef: # Only required when using service principal mode + # name: grafana-akv-creds # Only required when using service principal mode + +## Additional grafana server volume mounts +# Defines additional volume mounts. +extraVolumeMounts: [] + # - name: extra-volume-0 + # mountPath: /mnt/volume0 + # readOnly: true + # - name: extra-volume-1 + # mountPath: /mnt/volume1 + # readOnly: true + # - name: grafana-secrets + # mountPath: /mnt/volume2 + +## Additional Grafana server volumes +extraVolumes: [] + # - name: extra-volume-0 + # existingClaim: volume-claim + # - name: extra-volume-1 + # hostPath: + # path: /usr/shared/ + # type: "" + # - name: grafana-secrets + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "grafana-env-spc" + +## Container Lifecycle Hooks. Execute a specific bash command or make an HTTP request +lifecycleHooks: {} + # postStart: + # exec: + # command: [] + +## Pass the plugins you want installed as a list. +## +plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + ## You can also use other plugin download URL, as long as they are valid zip files, + ## and specify the name of the plugin after the semicolon. Like this: + # - https://grafana.com/api/plugins/marcusolsson-json-datasource/versions/1.3.2/download;marcusolsson-json-datasource + +## Configure grafana datasources +## ref: http://docs.grafana.org/administration/provisioning/#datasources +## +datasources: {} +# datasources.yaml: +# apiVersion: 1 +# datasources: +# - name: Prometheus +# type: prometheus +# url: http://prometheus-prometheus-server +# access: proxy +# isDefault: true +# - name: CloudWatch +# type: cloudwatch +# access: proxy +# uid: cloudwatch +# editable: false +# jsonData: +# authType: default +# defaultRegion: us-east-1 +# deleteDatasources: [] +# - name: Prometheus + +## Configure grafana alerting (can be templated) +## ref: http://docs.grafana.org/administration/provisioning/#alerting +## +alerting: {} + # rules.yaml: + # apiVersion: 1 + # groups: + # - orgId: 1 + # name: '{{ .Chart.Name }}_my_rule_group' + # folder: my_first_folder + # interval: 60s + # rules: + # - uid: my_id_1 + # title: my_first_rule + # condition: A + # data: + # - refId: A + # datasourceUid: '-100' + # model: + # conditions: + # - evaluator: + # params: + # - 3 + # type: gt + # operator: + # type: and + # query: + # params: + # - A + # reducer: + # type: last + # type: query + # datasource: + # type: __expr__ + # uid: '-100' + # expression: 1==0 + # intervalMs: 1000 + # maxDataPoints: 43200 + # refId: A + # type: math + # dashboardUid: my_dashboard + # panelId: 123 + # noDataState: Alerting + # for: 60s + # annotations: + # some_key: some_value + # labels: + # team: sre_team_1 + # contactpoints.yaml: + # secret: + # apiVersion: 1 + # contactPoints: + # - orgId: 1 + # name: cp_1 + # receivers: + # - uid: first_uid + # type: pagerduty + # settings: + # integrationKey: XXX + # severity: critical + # class: ping failure + # component: Grafana + # group: app-stack + # summary: | + # {{ `{{ include "default.message" . }}` }} + +## Configure notifiers +## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels +## +notifiers: {} +# notifiers.yaml: +# notifiers: +# - name: email-notifier +# type: email +# uid: email1 +# # either: +# org_id: 1 +# # or +# org_name: Main Org. +# is_default: true +# settings: +# addresses: an_email_address@example.com +# delete_notifiers: + +## Configure grafana dashboard providers +## ref: http://docs.grafana.org/administration/provisioning/#dashboards +## +## `path` must be /var/lib/grafana/dashboards/ +## +dashboardProviders: {} +# dashboardproviders.yaml: +# apiVersion: 1 +# providers: +# - name: 'default' +# orgId: 1 +# folder: '' +# type: file +# disableDeletion: false +# editable: true +# options: +# path: /var/lib/grafana/dashboards/default + +## Configure grafana dashboard to import +## NOTE: To use dashboards you must also enable/configure dashboardProviders +## ref: https://grafana.com/dashboards +## +## dashboards per provider, use provider name as key. +## +dashboards: {} + # default: + # some-dashboard: + # json: | + # $RAW_JSON + # custom-dashboard: + # file: dashboards/custom-dashboard.json + # prometheus-stats: + # gnetId: 2 + # revision: 2 + # datasource: Prometheus + # local-dashboard: + # url: https://example.com/repository/test.json + # token: '' + # local-dashboard-base64: + # url: https://example.com/repository/test-b64.json + # token: '' + # b64content: true + # local-dashboard-gitlab: + # url: https://example.com/repository/test-gitlab.json + # gitlabToken: '' + # local-dashboard-bitbucket: + # url: https://example.com/repository/test-bitbucket.json + # bearerToken: '' + # local-dashboard-azure: + # url: https://example.com/repository/test-azure.json + # basic: '' + # acceptHeader: '*/*' + +## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. +## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. +## ConfigMap data example: +## +## data: +## example-dashboard.json: | +## RAW_JSON +## +dashboardsConfigMaps: {} +# default: "" + +## Grafana's primary configuration +## NOTE: values in map will be converted to ini format +## ref: http://docs.grafana.org/installation/configuration/ +## +grafana.ini: + paths: + data: /var/lib/grafana/ + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net + server: + domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ .Values.ingress.hosts | first }}{{ else }}''{{ end }}" +## grafana Authentication can be enabled with the following values on grafana.ini + # server: + # The full public facing url you use in browser, used for redirects and emails + # root_url: + # https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana + # auth.github: + # enabled: false + # allow_sign_up: false + # scopes: user:email,read:org + # auth_url: https://github.com/login/oauth/authorize + # token_url: https://github.com/login/oauth/access_token + # api_url: https://api.github.com/user + # team_ids: + # allowed_organizations: + # client_id: + # client_secret: +## LDAP Authentication can be enabled with the following values on grafana.ini +## NOTE: Grafana will fail to start if the value for ldap.toml is invalid + # auth.ldap: + # enabled: true + # allow_sign_up: true + # config_file: /etc/grafana/ldap.toml + +## Grafana's LDAP configuration +## Templated by the template in _helpers.tpl +## NOTE: To enable the grafana.ini must be configured with auth.ldap.enabled +## ref: http://docs.grafana.org/installation/configuration/#auth-ldap +## ref: http://docs.grafana.org/installation/ldap/#configuration +ldap: + enabled: false + # `existingSecret` is a reference to an existing secret containing the ldap configuration + # for Grafana in a key `ldap-toml`. + existingSecret: "" + # `config` is the content of `ldap.toml` that will be stored in the created secret + config: "" + # config: |- + # verbose_logging = true + + # [[servers]] + # host = "my-ldap-server" + # port = 636 + # use_ssl = true + # start_tls = false + # ssl_skip_verify = false + # bind_dn = "uid=%s,ou=users,dc=myorg,dc=com" + +## Grafana's SMTP configuration +## NOTE: To enable, grafana.ini must be configured with smtp.enabled +## ref: http://docs.grafana.org/installation/configuration/#smtp +smtp: + # `existingSecret` is a reference to an existing secret containing the smtp configuration + # for Grafana. + existingSecret: "" + userKey: "user" + passwordKey: "password" + +## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders +## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards +sidecar: + image: + repository: rancher/mirrored-kiwigrid-k8s-sidecar + tag: 1.26.1 + sha: "" + imagePullPolicy: IfNotPresent + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + # skipTlsVerify Set to true to skip tls verification for kube api calls + # skipTlsVerify: true + enableUniqueFilenames: false + readinessProbe: {} + livenessProbe: {} + # Log level default for all sidecars. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. Defaults to INFO + # logLevel: INFO + alerts: + enabled: false + # Additional environment variables for the alerts sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with alert are marked with + label: grafana_alert + # value of label that the configmaps with alert are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for alert config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/alerting/reload" + # Absolute path to shell script to execute after a alert got reloaded + script: null + skipReload: false + # This is needed if skipReload is true, to load any alerts defined at startup time. + # Deploy the alert sidecar as an initContainer. + initAlerts: false + # Additional alert sidecar volume mounts + extraMounts: [] + # Sets the size limit of the alert sidecar emptyDir volume + sizeLimit: {} + dashboards: + enabled: false + # Additional environment variables for the dashboards sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + SCProvider: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + # value of label that the configmaps with dashboards are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) + folder: /tmp/dashboards + # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead + defaultFolderName: null + # Namespaces list. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces. + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # If specified, the sidecar will look for annotation with this name to create folder and put graph here. + # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. + folderAnnotation: null + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/dashboards/reload" + # Absolute path to shell script to execute after a configmap got reloaded + script: null + skipReload: false + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # provider configuration that lets grafana manage the dashboards + provider: + # name of the provider, should be unique + name: sidecarProvider + # orgid as configured in grafana + orgid: 1 + # folder in which the dashboards should be imported in grafana + folder: '' + # type of the provider + type: file + # disableDelete to activate a import-only behaviour + disableDelete: false + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + # allow Grafana to replicate dashboard structure from filesystem + foldersFromFilesStructure: false + # Additional dashboard sidecar volume mounts + extraMounts: [] + # Sets the size limit of the dashboard sidecar emptyDir volume + sizeLimit: {} + datasources: + enabled: false + # Additional environment variables for the datasourcessidecar + env: {} + envValueFrom: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with datasources are marked with + label: grafana_datasource + # value of label that the configmaps with datasources are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for datasource config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload datasources + reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + # Absolute path to shell script to execute after a datasource got reloaded + script: null + skipReload: true + # This is needed if skipReload is true, to load any datasources defined at startup time. + # Deploy the datasources sidecar as an initContainer. + initDatasources: true + # Sets the size limit of the datasource sidecar emptyDir volume + sizeLimit: {} + plugins: + enabled: false + # Additional environment variables for the plugins sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with plugins are marked with + label: grafana_plugin + # value of label that the configmaps with plugins are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for plugin config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload plugins + reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload" + # Absolute path to shell script to execute after a plugin got reloaded + script: null + skipReload: false + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any plugins defined at startup time. + initPlugins: false + # Sets the size limit of the plugin sidecar emptyDir volume + sizeLimit: {} + notifiers: + enabled: false + # Additional environment variables for the notifierssidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with notifiers are marked with + label: grafana_notifier + # value of label that the configmaps with notifiers are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for notifier config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload notifiers + reloadURL: "http://localhost:3000/api/admin/provisioning/notifications/reload" + # Absolute path to shell script to execute after a notifier got reloaded + script: null + skipReload: false + # Deploy the notifier sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any notifiers defined at startup time. + initNotifiers: false + # Sets the size limit of the notifier sidecar emptyDir volume + sizeLimit: {} + +## Override the deployment namespace +## +namespaceOverride: "" + +## Number of old ReplicaSets to retain +## +revisionHistoryLimit: 10 + +## Add a seperate remote image renderer deployment/service +imageRenderer: + deploymentStrategy: {} + # Enable the image-renderer deployment & service + enabled: false + replicas: 1 + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + image: + # image-renderer Image repository + repository: rancher/mirrored-grafana-grafana-image-renderer + # image-renderer Image tag + tag: 3.10.1 + # image-renderer Image sha (optional) + sha: "" + # image-renderer ImagePullPolicy + pullPolicy: Always + # extra environment variables + env: + HTTP_HOST: "0.0.0.0" + # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758 + # RENDERING_MODE: clustered + # IGNORE_HTTPS_ERRORS: true + + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + + # image-renderer deployment serviceAccount + serviceAccountName: "" + # image-renderer deployment securityContext + securityContext: {} + # image-renderer deployment container securityContext + containerSecurityContext: + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ['ALL'] + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ## image-renderer pod annotation + podAnnotations: {} + # image-renderer deployment Host Aliases + hostAliases: [] + # image-renderer deployment priority class + priorityClassName: '' + service: + # Enable the image-renderer service + enabled: true + # image-renderer service port name + portName: 'http' + # image-renderer service port used by both service and deployment + port: 8081 + targetPort: 8081 + # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 1m + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + # See: https://doc.crds.dev/github.com/prometheus-operator/kube-prometheus/monitoring.coreos.com/ServiceMonitor/v1@v0.11.0#spec-targetLabels + targetLabels: [] + # - targetLabel1 + # - targetLabel2 + # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana + grafanaProtocol: http + # In case a sub_path is used this needs to be added to the image renderer callback + grafanaSubPath: "" + # name of the image-renderer port on the pod + podPortName: http + # number of image-renderer replica sets to keep + revisionHistoryLimit: 10 + networkPolicy: + # Enable a NetworkPolicy to limit inbound traffic to only the created grafana pods + limitIngress: true + # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods + limitEgress: false + # Allow additional services to access image-renderer (eg. Prometheus operator when ServiceMonitor is enabled) + extraIngressSelectors: [] + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment (evaluated as template) + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: "default-scheduler" + +networkPolicy: + ## @param networkPolicy.enabled Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to grafana port defined. + ## When true, grafana will accept connections from any source + ## (with the correct destination port). + ## + ingress: true + ## @param networkPolicy.ingress When true enables the creation + ## an ingress network policy + ## + allowExternal: true + ## @param networkPolicy.explicitNamespacesSelector A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed + ## If explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the grafana. + ## But sometimes, we want the grafana to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + ## + ## + ## + ## + ## + ## + egress: + ## @param networkPolicy.egress.enabled When enabled, an egress network policy will be + ## created allowing grafana to connect to external data sources from kubernetes cluster. + enabled: false + ## + ## @param networkPolicy.egress.blockDNSResolution When enabled, DNS resolution will be blocked + ## for all pods in the grafana namespace. + blockDNSResolution: false + ## + ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress + ports: [] + ## Add ports to the egress by specifying - port: + ## E.X. + ## - port: 80 + ## - port: 443 + ## + ## @param networkPolicy.egress.to Allow egress traffic to specific destinations + to: [] + ## Add destinations to the egress by specifying - ipBlock: + ## E.X. + ## to: + ## - namespaceSelector: + ## matchExpressions: + ## - {key: role, operator: In, values: [grafana]} + ## + ## + ## + ## + ## + +# Enable backward compatibility of kubernetes where version below 1.13 doesn't have the enableServiceLinks option +enableKubeBackwardCompatibility: false +useStatefulSet: false +# Create a dynamic manifests via values: +extraObjects: [] + # - apiVersion: "kubernetes-client.io/v1" + # kind: ExternalSecret + # metadata: + # name: grafana-secrets + # spec: + # backendType: gcpSecretsManager + # data: + # - key: grafana-admin-password + # name: adminPassword + +# assertNoLeakedSecrets is a helper function defined in _helpers.tpl that checks if secret +# values are not exposed in the rendered grafana.ini configmap. It is enabled by default. +# +# To pass values into grafana.ini without exposing them in a configmap, use variable expansion: +# https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion +# +# Alternatively, if you wish to allow secret values to be exposed in the rendered grafana.ini configmap, +# you can disable this check by setting assertNoLeakedSecrets to false. +assertNoLeakedSecrets: true diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/Chart.yaml new file mode 100644 index 0000000000..acd648a5b4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedKubelet +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedKubelet/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/Chart.yaml new file mode 100644 index 0000000000..068932bacb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedNodeExporter +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/hardenedNodeExporter/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/Chart.yaml new file mode 100644 index 0000000000..275e02e5dc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: k3sServer +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/k3sServer/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/Chart.yaml new file mode 100644 index 0000000000..002a6a180d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/Chart.yaml @@ -0,0 +1,32 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-kube-state-metrics +apiVersion: v2 +appVersion: 2.10.1 +description: Install kube-state-metrics to generate and expose cluster-level metrics +home: https://github.com/kubernetes/kube-state-metrics/ +keywords: +- metric +- monitoring +- prometheus +- kubernetes +maintainers: +- email: tariq.ibrahim@mulesoft.com + name: tariq1890 +- email: manuel@rueg.eu + name: mrueg +- email: david@0xdc.me + name: dotdc +name: kube-state-metrics +sources: +- https://github.com/kubernetes/kube-state-metrics/ +type: application +version: 5.16.4 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/README.md new file mode 100644 index 0000000000..843be89e69 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/README.md @@ -0,0 +1,85 @@ +# kube-state-metrics Helm Chart + +Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics). + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Migrating from stable/kube-state-metrics and kubernetes/kube-state-metrics + +You can upgrade in-place: + +1. [get repository info](#get-repository-info) +1. [upgrade](#upgrading-chart) your existing release name using the new chart repository + +## Upgrading to v3.0.0 + +v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side. + +The upgraded chart now the following changes: + +* Dropped support for helm v2 (helm v3 or later is required) +* collectors key was renamed to resources +* namespace key was renamed to namespaces + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-state-metrics +``` + +### kube-rbac-proxy + +You can enable `kube-state-metrics` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy one RBAC proxy container per endpoint (metrics & telemetry). +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics-read +rules: + - apiGroups: [ "" ] + resources: ["services/kube-state-metrics"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/NOTES.txt new file mode 100644 index 0000000000..3589c24ec3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/NOTES.txt @@ -0,0 +1,23 @@ +kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. +The exposed metrics can be found here: +https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics + +The metrics are exported on the HTTP endpoint /metrics on the listening port. +In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-state-metrics.namespace" . }}.svc.cluster.local:{{ .Values.service.port }}/metrics + +They are served either as plaintext or protobuf depending on the Accept header. +They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint. + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints are now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "kube-state-metrics.fullname" . }}"] + verbs: + - get +``` +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl new file mode 100644 index 0000000000..ed277fbb53 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl @@ -0,0 +1,196 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "kube-state-metrics.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kube-state-metrics.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kube-state-metrics.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "kube-state-metrics.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-state-metrics.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kube-state-metrics.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "kube-state-metrics.labels" }} +helm.sh/chart: {{ template "kube-state-metrics.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" . }} +{{- include "kube-state-metrics.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kube-state-metrics.selectorLabels" }} +{{- if .Values.selectorOverride }} +{{ toYaml .Values.selectorOverride }} +{{- else }} +app.kubernetes.io/name: {{ include "kube-state-metrics.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "kube-state-metrics.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +The image to use for kube-state-metrics +*/}} +{{- define "kube-state-metrics.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +The image to use for kubeRBACProxy +*/}} +{{- define "kubeRBACProxy.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.kubeRBACProxy.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..025cd47a88 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + endpointSelector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + egress: + {{- if and .Values.networkPolicy.cilium .Values.networkPolicy.cilium.kubeApiServerSelector }} + {{ toYaml .Values.networkPolicy.cilium.kubeApiServerSelector | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + - port: {{ .Values.service.port | quote }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + - port: {{ .Values.selfMonitor.telemetryPort | default 8081 | quote }} + protocol: TCP + {{ end }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..cf9f628d04 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rbac.useClusterRole -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml new file mode 100644 index 0000000000..d38a75a51d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.customResourceState.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config.yaml: | + {{- toYaml .Values.customResourceState.config | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/deployment.yaml new file mode 100644 index 0000000000..03158eb948 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/deployment.yaml @@ -0,0 +1,314 @@ +apiVersion: apps/v1 +{{- if .Values.autosharding.enabled }} +kind: StatefulSet +{{- else }} +kind: Deployment +{{- end }} +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: +{{ toYaml .Values.annotations | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + replicas: {{ .Values.replicas }} + {{- if not .Values.autosharding.enabled }} + strategy: + type: {{ .Values.updateStrategy | default "RollingUpdate" }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.autosharding.enabled }} + serviceName: {{ template "kube-state-metrics.fullname" . }} + volumeClaimTemplates: [] + {{- end }} + template: + metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 8 }} + {{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + spec: + hostNetwork: {{ .Values.hostNetwork }} + serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 6 }} + {{- end }} + containers: + {{- $servicePort := ternary 9090 (.Values.service.port | default 8080) .Values.kubeRBACProxy.enabled}} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - name: {{ template "kube-state-metrics.name" . }} + {{- if .Values.autosharding.enabled }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + args: + {{- if .Values.extraArgs }} + {{- .Values.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --port={{ $servicePort }} + {{- if .Values.collectors }} + - --resources={{ .Values.collectors | join "," }} + {{- end }} + {{- if .Values.metricLabelsAllowlist }} + - --metric-labels-allowlist={{ .Values.metricLabelsAllowlist | join "," }} + {{- end }} + {{- if .Values.metricAnnotationsAllowList }} + - --metric-annotations-allowlist={{ .Values.metricAnnotationsAllowList | join "," }} + {{- end }} + {{- if .Values.metricAllowlist }} + - --metric-allowlist={{ .Values.metricAllowlist | join "," }} + {{- end }} + {{- if .Values.metricDenylist }} + - --metric-denylist={{ .Values.metricDenylist | join "," }} + {{- end }} + {{- $namespaces := list }} + {{- if .Values.namespaces }} + {{- range $ns := join "," .Values.namespaces | split "," }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + {{- if .Values.releaseNamespace }} + {{- $namespaces = append $namespaces ( include "kube-state-metrics.namespace" . ) }} + {{- end }} + {{- if $namespaces }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + {{- if .Values.namespacesDenylist }} + - --namespaces-denylist={{ tpl (.Values.namespacesDenylist | join ",") $ }} + {{- end }} + {{- if .Values.autosharding.enabled }} + - --pod=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + {{- end }} + {{- if .Values.kubeconfig.enabled }} + - --kubeconfig=/opt/k8s/.kube/config + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - --telemetry-host=127.0.0.1 + - --telemetry-port={{ $telemetryPort }} + {{- else }} + {{- if .Values.selfMonitor.telemetryHost }} + - --telemetry-host={{ .Values.selfMonitor.telemetryHost }} + {{- end }} + {{- if .Values.selfMonitor.telemetryPort }} + - --telemetry-port={{ $telemetryPort }} + {{- end }} + {{- end }} + {{- if .Values.customResourceState.enabled }} + - --custom-resource-state-config-file=/etc/customresourcestate/config.yaml + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumeMounts) }} + volumeMounts: + {{- if .Values.kubeconfig.enabled }} + - name: kubeconfig + mountPath: /opt/k8s/.kube/ + readOnly: true + {{- end }} + {{- if .Values.customResourceState.enabled }} + - name: customresourcestate-config + mountPath: /etc/customresourcestate + readOnly: true + {{- end }} + {{- if .Values.volumeMounts }} +{{ toYaml .Values.volumeMounts | indent 8 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + image: {{ include "kube-state-metrics.image" . }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + {{- if .Values.selfMonitor.enabled }} + - containerPort: {{ $telemetryPort }} + name: "metrics" + {{- end }} + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: /healthz + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- if .Values.resources }} + resources: +{{ toYaml .Values.resources | indent 10 }} +{{- end }} +{{- if .Values.containerSecurityContext }} + securityContext: +{{ toYaml .Values.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-http + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port | default 8080}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.selfMonitor.enabled }} + - name: kube-rbac-proxy-telemetry + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.selfMonitor.telemetryPort | default 8081 }} + - --upstream=http://127.0.0.1:{{ $telemetryPort }}/ + - --proxy-endpoints-port=8889 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + name: "metrics" + - containerPort: 8889 + name: "metrics-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8889 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- end }} + {{- end }} + {{- with .Values.containers }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.topologySpreadConstraints | indent 8 }} + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumes) (.Values.kubeRBACProxy.enabled) }} + volumes: + {{- if .Values.kubeconfig.enabled}} + - name: kubeconfig + secret: + secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + {{- end }} + {{- if .Values.kubeRBACProxy.enabled}} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + {{- end }} + {{- if .Values.customResourceState.enabled}} + - name: customresourcestate-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + {{- end }} + {{- if .Values.volumes }} +{{ toYaml .Values.volumes | indent 8 }} + {{- end }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml new file mode 100644 index 0000000000..6af0084502 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.kubeconfig.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +type: Opaque +data: + config: '{{ .Values.kubeconfig.secret }}' +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml new file mode 100644 index 0000000000..309b38ec54 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "kubernetes") }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + {{- if .Values.networkPolicy.egress }} + ## Deny all egress by default + egress: + {{- toYaml .Values.networkPolicy.egress | nindent 4 }} + {{- end }} + ingress: + {{- if .Values.networkPolicy.ingress }} + {{- toYaml .Values.networkPolicy.ingress | nindent 4 }} + {{- else }} + ## Allow ingress on default ports by default + - ports: + - port: {{ .Values.service.port | default 8080 }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - port: {{ $telemetryPort }} + protocol: TCP + {{- end }} + {{- end }} + podSelector: + {{- if .Values.networkPolicy.podSelector }} + {{- toYaml .Values.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + policyTypes: + - Ingress + - Egress +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/pdb.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/pdb.yaml new file mode 100644 index 0000000000..3771b511de --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podDisruptionBudget -}} +{{ if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" -}} +apiVersion: policy/v1 +{{- else -}} +apiVersion: policy/v1beta1 +{{- end }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..d9d944d740 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +{{- if .Values.podSecurityPolicy.annotations }} + annotations: +{{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }} +{{- end }} +spec: + privileged: false + volumes: + - 'secret' +{{- if .Values.podSecurityPolicy.additionalVolumes }} +{{ toYaml .Values.podSecurityPolicy.additionalVolumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..c69e01a716 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..df81c49028 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml new file mode 100644 index 0000000000..671dc9d660 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml @@ -0,0 +1,22 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "kube-state-metrics.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "kube-state-metrics.fullname" . }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/role.yaml new file mode 100644 index 0000000000..0170878376 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/role.yaml @@ -0,0 +1,215 @@ +{{- if not (kindIs "slice" .Values.collectors) }} +{{- fail "Collectors need to be a List since kube-state-metrics chart 3.2.2. Please check README for more information."}} +{{- end }} +{{- if and (eq .Values.rbac.create true) (not .Values.rbac.useExistingRole) -}} +{{- range (ternary (join "," .Values.namespaces | split "," ) (list "") (eq $.Values.rbac.useClusterRole false)) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +{{- if eq $.Values.rbac.useClusterRole false }} +kind: Role +{{- else }} +kind: ClusterRole +{{- end }} +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- if eq $.Values.rbac.useClusterRole false }} + namespace: {{ . }} +{{- end }} +rules: +{{ if has "certificatesigningrequests" $.Values.collectors }} +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] +{{ end -}} +{{ if has "configmaps" $.Values.collectors }} +- apiGroups: [""] + resources: + - configmaps + verbs: ["list", "watch"] +{{ end -}} +{{ if has "cronjobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - cronjobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "daemonsets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "deployments" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpoints" $.Values.collectors }} +- apiGroups: [""] + resources: + - endpoints + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpointslices" $.Values.collectors }} +- apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: ["list", "watch"] +{{ end -}} +{{ if has "horizontalpodautoscalers" $.Values.collectors }} +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "ingresses" $.Values.collectors }} +- apiGroups: ["extensions", "networking.k8s.io"] + resources: + - ingresses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "jobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "leases" $.Values.collectors }} +- apiGroups: ["coordination.k8s.io"] + resources: + - leases + verbs: ["list", "watch"] +{{ end -}} +{{ if has "limitranges" $.Values.collectors }} +- apiGroups: [""] + resources: + - limitranges + verbs: ["list", "watch"] +{{ end -}} +{{ if has "mutatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "namespaces" $.Values.collectors }} +- apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch"] +{{ end -}} +{{ if has "networkpolicies" $.Values.collectors }} +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: ["list", "watch"] +{{ end -}} +{{ if has "nodes" $.Values.collectors }} +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumeclaims" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumes" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "poddisruptionbudgets" $.Values.collectors }} +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "pods" $.Values.collectors }} +- apiGroups: [""] + resources: + - pods + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicasets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicationcontrollers" $.Values.collectors }} +- apiGroups: [""] + resources: + - replicationcontrollers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "resourcequotas" $.Values.collectors }} +- apiGroups: [""] + resources: + - resourcequotas + verbs: ["list", "watch"] +{{ end -}} +{{ if has "secrets" $.Values.collectors }} +- apiGroups: [""] + resources: + - secrets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "services" $.Values.collectors }} +- apiGroups: [""] + resources: + - services + verbs: ["list", "watch"] +{{ end -}} +{{ if has "statefulsets" $.Values.collectors }} +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "storageclasses" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "validatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "volumeattachments" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - volumeattachments + verbs: ["list", "watch"] +{{ end -}} +{{- if $.Values.kubeRBACProxy.enabled }} +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] +{{- end }} +{{- if $.Values.customResourceState.enabled }} +- apiGroups: ["apiextensions.k8s.io"] + resources: + - customresourcedefinitions + verbs: ["list", "watch"] +{{- end }} +{{ if $.Values.rbac.extraRules }} +{{ toYaml $.Values.rbac.extraRules }} +{{ end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml new file mode 100644 index 0000000000..330651b73f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.rbac.useClusterRole false) -}} +{{- range (join "," $.Values.namespaces) | split "," }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} + namespace: {{ . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role +{{- if (not $.Values.rbac.useExistingRole) }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- else }} + name: {{ $.Values.rbac.useExistingRole }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" $ }} + namespace: {{ template "kube-state-metrics.namespace" $ }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/service.yaml new file mode 100644 index 0000000000..6c486a662a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/service.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + annotations: + {{- if .Values.prometheusScrape }} + prometheus.io/scrape: '{{ .Values.prometheusScrape }}' + {{- end }} + {{- if .Values.service.annotations }} + {{- toYaml .Values.service.annotations | nindent 4 }} + {{- end }} +spec: + type: "{{ .Values.service.type }}" + ports: + - name: "http" + protocol: TCP + port: {{ .Values.service.port | default 8080}} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port | default 8080}} + {{ if .Values.selfMonitor.enabled }} + - name: "metrics" + protocol: TCP + port: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + targetPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- if .Values.selfMonitor.telemetryNodePort }} + nodePort: {{ .Values.selfMonitor.telemetryNodePort }} + {{- end }} + {{ end }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if .Values.autosharding.enabled }} + clusterIP: None +{{- else if .Values.service.clusterIP }} + clusterIP: "{{ .Values.service.clusterIP }}" +{{- end }} + selector: + {{- include "kube-state-metrics.selectorLabels" . | indent 4 }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml new file mode 100644 index 0000000000..38a93b31d1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- if .Values.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml new file mode 100644 index 0000000000..01ec44e067 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -0,0 +1,126 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + {{- with .Values.prometheus.monitor.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- with .Values.prometheus.monitor.targetLabels }} + targetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | indent 2 }} + {{- if .Values.prometheus.monitor.namespaceSelector }} + namespaceSelector: + matchNames: + {{- with .Values.prometheus.monitor.namespaceSelector }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + endpoints: + - port: http + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + metricRelabelings: + {{- if .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml .Values.prometheus.monitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.selfMonitor.enabled }} + - port: metrics + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml new file mode 100644 index 0000000000..489de147c1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} + resources: + - statefulsets + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml new file mode 100644 index 0000000000..73b37a4f64 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..f46305b517 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml @@ -0,0 +1,44 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-state-metrics.name" . }} + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{ toYaml .Values.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{ toYaml .Values.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + {{- if .Values.autosharding.enabled }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ template "kube-state-metrics.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/values.yaml new file mode 100644 index 0000000000..bc8ee28fda --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kube-state-metrics/values.yaml @@ -0,0 +1,491 @@ +# Default values for kube-state-metrics. +prometheusScrape: true +image: + registry: docker.io + repository: rancher/mirrored-kube-state-metrics-kube-state-metrics + tag: v2.10.1 + sha: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: "image-pull-secret" + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "" + +# If set to true, this will deploy kube-state-metrics as a StatefulSet and the data +# will be automatically sharded across <.Values.replicas> pods using the built-in +# autodiscovery feature: https://github.com/kubernetes/kube-state-metrics#automated-sharding +# This is an experimental feature and there are no stability guarantees. +autosharding: + enabled: false + +replicas: 1 + +# Change the deployment strategy when autosharding is disabled. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +# The default is "RollingUpdate" as per Kubernetes defaults. +# During a release, 'RollingUpdate' can lead to two running instances for a short period of time while 'Recreate' can create a small gap in data. +# updateStrategy: Recreate + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +# List of additional cli arguments to configure kube-state-metrics +# for example: --enable-gzip-encoding, --log-file, etc. +# all the possible args can be found here: https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md +extraArgs: [] + +service: + port: 8080 + # Default to clusterIP for backward compatibility + type: ClusterIP + nodePort: 0 + loadBalancerIP: "" + # Only allow access to the loadBalancerIP from these IPs + loadBalancerSourceRanges: [] + clusterIP: "" + annotations: {} + +## Additional labels to add to all resources +customLabels: {} + # app: kube-state-metrics + +## Override selector labels +selectorOverride: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +hostNetwork: false + +rbac: + # If true, create & use RBAC resources + create: true + + # Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to it, rolename set here. + # useExistingRole: your-existing-role + + # If set to false - Run without Cluteradmin privs needed - ONLY works if namespace is also set (if useExistingRole is set this name is used as ClusterRole or Role to bind to) + useClusterRole: true + + # Add permissions for CustomResources' apiGroups in Role/ClusterRole. Should be used in conjunction with Custom Resource State Metrics configuration + # Example: + # - apiGroups: ["monitoring.coreos.com"] + # resources: ["prometheuses"] + # verbs: ["list", "watch"] + extraRules: [] + +# Configure kube-rbac-proxy. When enabled, creates one kube-rbac-proxy container per exposed HTTP endpoint (metrics and telemetry if enabled). +# The requests are served through the same service but requests are then HTTPS. +kubeRBACProxy: + enabled: false + image: + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.14.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-prxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + + ## volumeMounts enables mounting custom volumes in rbac-proxy containers + ## Useful for TLS certificates and keys + volumeMounts: [] + # - mountPath: /etc/tls + # name: kube-rbac-proxy-tls + # readOnly: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created, require rbac true + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # Reference to one or more secrets to be used when pulling images + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + imagePullSecrets: [] + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +prometheus: + monitor: + enabled: false + annotations: {} + additionalLabels: {} + namespace: "" + namespaceSelector: [] + jobLabel: "" + targetLabels: [] + podTargetLabels: [] + interval: "" + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + scrapeTimeout: "" + proxyUrl: "" + ## Whether to enable HTTP2 for servicemonitor + # enableHttp2: false + selectorOverride: {} + honorLabels: false + metricRelabelings: [] + relabelings: [] + scheme: "" + ## File to read bearer token for scraping targets + bearerTokenFile: "" + ## Secret to mount to read bearer token for scraping targets. The secret needs + ## to be in the same namespace as the service monitor and accessible by the + ## Prometheus Operator + bearerTokenSecret: {} + # name: secret-name + # key: key-name + tlsConfig: {} + +## Specify if a Pod Security Policy for kube-state-metrics must be created +## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +podSecurityPolicy: + annotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + additionalVolumes: [] + +## Configure network policy for kube-state-metrics +networkPolicy: + enabled: false + # networkPolicy.flavor -- Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + ## Configure the cilium network policy kube-apiserver selector + # cilium: + # kubeApiServerSelector: + # - toEntities: + # - kube-apiserver + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app.kubernetes.io/name: kube-state-metrics + +securityContext: + enabled: true + runAsGroup: 65534 + runAsUser: 65534 + fsGroup: 65534 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +## Specify security settings for a Container +## Allows overrides and additional options compared to (Pod) securityContext +## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +## Affinity settings for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ +affinity: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +## Topology spread constraints for pod assignment +## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +topologySpreadConstraints: [] + +# Annotations to be added to the deployment/statefulset +annotations: {} + +# Annotations to be added to the pod +podAnnotations: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +# Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} + +# Comma-separated list of metrics to be exposed. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricAllowlist: [] + +# Comma-separated list of metrics not to be enabled. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricDenylist: [] + +# Comma-separated list of additional Kubernetes label keys that will be used in the resource's +# labels metric. By default the metric contains only name and namespace labels. +# To include additional labels, provide a list of resource names in their plural form and Kubernetes +# label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. +# A single '*' can be provided per resource instead to allow any labels, but that has +# severe performance implications (Example: '=pods=[*]'). +metricLabelsAllowlist: [] + # - namespaces=[k8s-label-1,k8s-label-n] + +# Comma-separated list of Kubernetes annotations keys that will be used in the resource' +# labels metric. By default the metric contains only name and namespace labels. +# To include additional annotations provide a list of resource names in their plural form and Kubernetes +# annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. +# A single '*' can be provided per resource instead to allow any annotations, but that has +# severe performance implications (Example: '=pods=[*]'). +metricAnnotationsAllowList: [] + # - pods=[k8s-annotation-1,k8s-annotation-n] + +# Available collectors for kube-state-metrics. +# By default, all available resources are enabled, comment out to disable. +collectors: + - certificatesigningrequests + - configmaps + - cronjobs + - daemonsets + - deployments + - endpoints + - horizontalpodautoscalers + - ingresses + - jobs + - leases + - limitranges + - mutatingwebhookconfigurations + - namespaces + - networkpolicies + - nodes + - persistentvolumeclaims + - persistentvolumes + - poddisruptionbudgets + - pods + - replicasets + - replicationcontrollers + - resourcequotas + - secrets + - services + - statefulsets + - storageclasses + - validatingwebhookconfigurations + - volumeattachments + +# Enabling kubeconfig will pass the --kubeconfig argument to the container +kubeconfig: + enabled: false + # base64 encoded kube-config file + secret: + +# Enabling support for customResourceState, will create a configMap including your config that will be read from kube-state-metrics +customResourceState: + enabled: false + # Add (Cluster)Role permissions to list/watch the customResources defined in the config to rbac.extraRules + config: {} + +# Enable only the release namespace for collecting resources. By default all namespaces are collected. +# If releaseNamespace and namespaces are both set a merged list will be collected. +releaseNamespace: false + +# Comma-separated list(string) or yaml list of namespaces to be enabled for collecting resources. By default all namespaces are collected. +namespaces: "" + +# Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, +# only namespaces that are excluded in namespaces-denylist will be used. +namespacesDenylist: "" + +## Override the deployment namespace +## +namespaceOverride: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +## Provide a k8s version to define apiGroups for podSecurityPolicy Cluster Role. +## For example: kubeTargetVersionOverride: 1.14.9 +## +kubeTargetVersionOverride: "" + +# Enable self metrics configuration for service and Service Monitor +# Default values for telemetry configuration can be overridden +# If you set telemetryNodePort, you must also set service.type to NodePort +selfMonitor: + enabled: false + # telemetryHost: 0.0.0.0 + # telemetryPort: 8081 + # telemetryNodePort: 0 + +# Enable vertical pod autoscaler support for kube-state-metrics +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: [] + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# volumeMounts are used to add custom volume mounts to deployment. +# See example below +volumeMounts: [] +# - mountPath: /etc/config +# name: config-volume + +# volumes are used to add custom volumes to deployment +# See example below +volumes: [] +# - configMap: +# name: cm-for-volume +# name: config-volume + +# Extra manifests to deploy as an array +extraManifests: [] + # - apiVersion: v1 + # kind: ConfigMap + # metadata: + # labels: + # name: prometheus-extra + # data: + # extra-data: "value" + +## Containers allows injecting additional containers. +containers: [] + # - name: crd-init + # image: kiwigrid/k8s-sidecar:latest + +## InitContainers allows injecting additional initContainers. +initContainers: [] + # - name: crd-sidecar + # image: kiwigrid/k8s-sidecar:latest + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/Chart.yaml new file mode 100644 index 0000000000..ad9fba247a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/Chart.yaml new file mode 100644 index 0000000000..d144d3ee2a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmEtcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/Chart.yaml new file mode 100644 index 0000000000..a1222c4feb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmProxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/Chart.yaml new file mode 100644 index 0000000000..78a44159a9 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmScheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/kubeAdmScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/Chart.yaml new file mode 100644 index 0000000000..d067725a17 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/Chart.yaml @@ -0,0 +1,28 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-prometheus-adapter +apiVersion: v1 +appVersion: v0.10.0 +description: A Helm chart for k8s prometheus adapter +home: https://github.com/kubernetes-sigs/prometheus-adapter +keywords: +- hpa +- metrics +- prometheus +- adapter +kubeVersion: '>=1.26.0-0' +maintainers: +- email: mattias.gees@jetstack.io + name: mattiasgees +- name: steven-sheehy +- email: hfernandez@mesosphere.com + name: hectorj2f +name: prometheus-adapter +sources: +- https://github.com/kubernetes/charts +- https://github.com/kubernetes-sigs/prometheus-adapter +version: 4.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/README.md new file mode 100644 index 0000000000..d77bb0c920 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/README.md @@ -0,0 +1,160 @@ +# Prometheus Adapter + +Installs the [Prometheus Adapter](https://github.com/kubernetes-sigs/prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics. + +## Prerequisites + +Kubernetes 1.14+ + +## Get Helm Repositories Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-adapter +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Helm Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### To 4.2.0 + +Readiness and liveness probes are now fully configurable through values `readinessProbe` and `livenessProbe`. The previous values have been kept as defaults. + +### To 4.0.0 + +Previously, security context of the container was set directly in the deployment template. This release makes it configurable through the new configuration variable `securityContext` whilst keeping the previously set values as defaults. Furthermore, previous variable `runAsUser` is now set in `securityContext` and is not used any longer. Please, use `securityContext.runAsUser` instead. In the same security context, `seccompProfile` has been enabled and set to type `RuntimeDefault`. + +### To 3.0.0 + +Due to a change in deployment labels, the upgrade requires `helm upgrade --force` in order to re-create the deployment. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-adapter +``` + +### Prometheus Service Endpoint + +To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`. + +### Adapter Rules + +Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/kubernetes-sigs/prometheus-adapter/blob/master/docs/config.md) for the proper format). + +### Horizontal Pod Autoscaler Metrics + +Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics). + +The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis): + +### Custom Metrics + +Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`: + +```yaml +rules: + custom: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### External Metrics + +Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`: + +```yaml +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_external_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### Resource Metrics + +Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`: + +```yaml +rules: + resource: + cpu: + containerQuery: | + sum by (<<.GroupBy>>) ( + rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + - + avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m +``` + +**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/NOTES.txt new file mode 100644 index 0000000000..b7b9b99322 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/NOTES.txt @@ -0,0 +1,9 @@ +{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed. +In a few minutes you should be able to list metrics using the following command(s): +{{ if .Values.rules.resource }} + kubectl get --raw /apis/metrics.k8s.io/v1beta1 +{{- end }} + kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 +{{ if .Values.rules.external }} + kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl new file mode 100644 index 0000000000..edbb829b2b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl @@ -0,0 +1,113 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-prometheus-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-prometheus-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "k8s-prometheus-adapter.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-prometheus-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "k8s-prometheus-adapter.labels" }} +helm.sh/chart: {{ include "k8s-prometheus-adapter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "k8s-prometheus-adapter.name" . }} +{{- include "k8s-prometheus-adapter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "k8s-prometheus-adapter.selectorLabels" }} +app.kubernetes.io/name: {{ include "k8s-prometheus-adapter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8s-prometheus-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "k8s-prometheus-adapter.pdb.apiVersion" -}} +{{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} + {{- print "policy/v1" -}} +{{- else -}} + {{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml new file mode 100644 index 0000000000..4e32c964c6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml @@ -0,0 +1,76 @@ +{{- if .Values.certManager.enabled -}} +--- +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + duration: {{ .Values.certManager.caCertDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.prometheus-adapter" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + ca: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert +--- +# Finally, generate a serving certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + duration: {{ .Values.certManager.certDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + dnsNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }}.svc +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml new file mode 100644 index 0000000000..6701e6ba08 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-system-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml new file mode 100644 index 0000000000..67efd2aa2f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml new file mode 100644 index 0000000000..2c690a03cc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/configmap.yaml new file mode 100644 index 0000000000..17f415d970 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/configmap.yaml @@ -0,0 +1,97 @@ +{{- if not .Values.rules.existing -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +data: + config.yaml: | +{{- if or .Values.rules.default .Values.rules.custom }} + rules: +{{- if .Values.rules.default }} + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{- end -}} +{{- if .Values.rules.custom }} +{{ toYaml .Values.rules.custom | indent 4 }} +{{- end -}} +{{- end -}} +{{- if .Values.rules.external }} + externalRules: +{{ toYaml .Values.rules.external | indent 4 }} +{{- end -}} +{{- if .Values.rules.resource }} + resourceRules: +{{ toYaml .Values.rules.resource | indent 6 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml new file mode 100644 index 0000000000..8b7b4e555e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if or .Values.rules.default .Values.rules.custom }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: custom.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..0cc6920836 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,24 @@ +{{- /* +This if must be aligned with custom-metrics-cluster-role.yaml +as otherwise this binding will point to not existing role. +*/ -}} +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml new file mode 100644 index 0000000000..4aa15ffe99 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: ["*"] + verbs: ["*"] +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/deployment.yaml new file mode 100644 index 0000000000..a7ea3310a0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/deployment.yaml @@ -0,0 +1,143 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + {{- if or .Values.customAnnotations .Values.deploymentAnnotations }} + annotations: + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.deploymentAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + replicas: {{ .Values.replicas }} + strategy: {{ toYaml .Values.strategy | nindent 4 }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} + template: + metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + name: {{ template "k8s-prometheus-adapter.name" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy }} + {{- end}} + containers: + - name: {{ .Chart.Name }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 8 }} + {{- end }} + args: + - /adapter + - --secure-port={{ .Values.listenPort }} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - --tls-cert-file=/var/run/serving-cert/tls.crt + - --tls-private-key-file=/var/run/serving-cert/tls.key + {{- end }} + - --cert-dir=/tmp/cert + - --prometheus-url={{ tpl .Values.prometheus.url . }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}}{{ .Values.prometheus.path }} + - --metrics-relist-interval={{ .Values.metricsRelistInterval }} + - --v={{ .Values.logLevel }} + - --config=/etc/adapter/config.yaml + {{- if .Values.extraArguments }} + {{- toYaml .Values.extraArguments | trim | nindent 8 }} + {{- end }} + ports: + - containerPort: {{ .Values.listenPort }} + name: https + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }} + {{ end }} + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} + priorityClassName: {{ .Values.priorityClassName }} + {{- if .Values.podSecurityContext }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | trim | nindent 6 }} + {{ end }} + - name: config + configMap: + name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }} + - name: tmp + emptyDir: {} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - name: volume-serving-cert + secret: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml new file mode 100644 index 0000000000..21339af128 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.external }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: external.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..05547bd323 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml new file mode 100644 index 0000000000..212ea78b25 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/pdb.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/pdb.yaml new file mode 100644 index 0000000000..205761a9f1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/pdb.yaml @@ -0,0 +1,23 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: {{ include "k8s-prometheus-adapter.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/psp.yaml new file mode 100644 index 0000000000..fded5a7491 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/psp.yaml @@ -0,0 +1,66 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.psp.create) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + hostPorts: + - min: {{ .Values.listenPort }} + max: {{ .Values.listenPort }} + {{- end }} + fsGroup: + rule: RunAsAny + runAsGroup: + rule: RunAsAny + runAsUser: + rule: MustRunAs + ranges: + - min: 1024 + max: 65535 + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - secret + - emptyDir + - configMap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +rules: +- apiGroups: + - 'policy' + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml new file mode 100644 index 0000000000..0cc9fff6a2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.resource}} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml new file mode 100644 index 0000000000..3c247e48d2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml new file mode 100644 index 0000000000..73d8953046 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml new file mode 100644 index 0000000000..d3c77c1c65 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/secret.yaml new file mode 100644 index 0000000000..3e7e8887bd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- if .Values.tls.enable -}} +apiVersion: v1 +kind: Secret +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +type: kubernetes.io/tls +data: + tls.crt: {{ b64enc .Values.tls.certificate }} + tls.key: {{ b64enc .Values.tls.key }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/service.yaml new file mode 100644 index 0000000000..ddac37cfa1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + {{- if or .Values.service.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.service.annotations }} + {{ toYaml .Values.service.annotations | indent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + ports: + - port: {{ .Values.service.port }} + protocol: TCP + targetPort: https + selector: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 4 }} + type: {{ .Values.service.type }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..30a169ae0e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +{{- if or .Values.serviceAccount.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.serviceAccount.annotations }} + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/values.yaml new file mode 100644 index 0000000000..a1445a23f1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-adapter/values.yaml @@ -0,0 +1,277 @@ +# Default values for k8s-prometheus-adapter.. +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +affinity: {} + +topologySpreadConstraints: [] + +image: + repository: rancher/mirrored-prometheus-adapter-prometheus-adapter + tag: v0.12.0 + pullPolicy: IfNotPresent + +logLevel: 4 + +metricsRelistInterval: 1m + +listenPort: 6443 + +nodeSelector: {} + +priorityClassName: "" + +## Override the release namespace (for multi-namespace deployments in combined charts) +namespaceOverride: "" + +## Additional annotations to add to all resources +customAnnotations: {} + # role: custom-metrics + +## Additional labels to add to all resources +customLabels: {} + # monitoring: prometheus-adapter + +# Url to access prometheus +prometheus: + # Value is templated + url: http://prometheus.default.svc + port: 9090 + path: "" + +replicas: 1 + +# k8s 1.21 needs fsGroup to be set for non root deployments +# ref: https://github.com/kubernetes/kubernetes/issues/70679 +podSecurityContext: + fsGroup: 10001 + +# SecurityContext of the container +# ref. https://kubernetes.io/docs/tasks/configure-pod-container/security-context +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["all"] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + +rbac: + # Specifies whether RBAC resources should be created + create: true + +psp: + # Specifies whether PSP resources should be created + create: false + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +# Custom DNS configuration to be added to prometheus-adapter pods +dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + +# Configure liveness probe +# https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#Probe +livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +# Configure readiness probe +readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +rules: + default: true + + custom: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_custom_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # Mounts a configMap with pre-generated rules for use. Overrides the + # default, custom, external and resource entries + existing: + + external: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_external_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # resource: + # cpu: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # memory: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + # - + # avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # window: 3m + +service: + annotations: {} + port: 443 + type: ClusterIP + # clusterIP: 1.2.3.4 + +tls: + enable: false + ca: |- + # Public CA file that signed the APIService + key: |- + # Private key of the APIService + certificate: |- + # Public key of the APIService + +# Set environment variables from secrets, configmaps or by setting them as name/value +env: [] + # - name: TMP_DIR + # value: /tmp + # - name: PASSWORD + # valueFrom: + # secretKeyRef: + # name: mysecret + # key: password + # optional: false + +# Any extra arguments +extraArguments: [] + # - --tls-private-key-file=/etc/tls/tls.key + # - --tls-cert-file=/etc/tls/tls.crt + +# Any extra volumes +extraVolumes: [] + # - name: example-name + # hostPath: + # path: /path/on/host + # type: DirectoryOrCreate + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + # type: File + +# Any extra volume mounts +extraVolumeMounts: [] + # - name: example-name + # mountPath: /path/in/container + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +tolerations: [] + +# Labels added to the pod +podLabels: {} + +# Annotations added to the pod +podAnnotations: {} + +# Annotations added to the deployment +deploymentAnnotations: {} + +hostNetwork: + # Specifies if prometheus-adapter should be started in hostNetwork mode. + # + # You would require this enabled if you use alternate overlay networking for pods and + # API server unable to communicate with metrics-server. As an example, this is required + # if you use Weave network on EKS. See also dnsPolicy + enabled: false + +# When hostNetwork is enabled, you probably want to set this to ClusterFirstWithHostNet +# dnsPolicy: ClusterFirstWithHostNet + +# Deployment strategy type +strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% + +podDisruptionBudget: + # Specifies if PodDisruptionBudget should be enabled + # When enabled, minAvailable or maxUnavailable should also be defined. + enabled: false + minAvailable: + maxUnavailable: 1 + +certManager: + enabled: false + caCertDuration: 43800h + certDuration: 8760h diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/Chart.yaml new file mode 100644 index 0000000000..9130cbcc91 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/Chart.yaml @@ -0,0 +1,25 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: |- + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts +apiVersion: v2 +appVersion: 1.7.0 +description: A Helm chart for prometheus node-exporter +home: https://github.com/prometheus/node_exporter/ +keywords: +- node-exporter +- prometheus +- exporter +maintainers: +- email: gianrubio@gmail.com + name: gianrubio +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rootsandtrees@posteo.de + name: zeritti +name: prometheus-node-exporter +sources: +- https://github.com/prometheus/node_exporter/ +type: application +version: 4.30.3 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/README.md new file mode 100644 index 0000000000..149b982267 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/README.md @@ -0,0 +1,97 @@ + +# Prometheus Node Exporter + +Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a Prometheus [Node Exporter](http://github.com/prometheus/node_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-node-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/prometheus-node-exporter --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### 3.x to 4.x + +Starting from version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i prometheus-node-exporter prometheus-community/prometheus-node-exporter +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 2.x to 3.x + +Change the following: + +```yaml +hostRootFsMount: true +``` + +to: + +```yaml +hostRootFsMount: + enabled: true + mountPropagation: HostToContainer +``` + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-node-exporter +``` + +### kube-rbac-proxy + +You can enable `prometheus-node-exporter` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy a RBAC proxy container protecting the node-exporter endpoint. +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-node-exporter-read +rules: + - apiGroups: [ "" ] + resources: ["services/node-exporter-prometheus-node-exporter"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt new file mode 100644 index 0000000000..8c5391f1f7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt @@ -0,0 +1,29 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus-node-exporter.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "prometheus-node-exporter.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "prometheus-node-exporter.namespace" . }} {{ template "prometheus-node-exporter.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "prometheus-node-exporter.namespace" . }} -l "app.kubernetes.io/name={{ template "prometheus-node-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:{{ .Values.service.port }} to use your application" + kubectl port-forward --namespace {{ template "prometheus-node-exporter.namespace" . }} $POD_NAME {{ .Values.service.port }} +{{- end }} + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints is now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "prometheus-node-exporter.fullname" . }}"] + verbs: + - get +``` +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl new file mode 100644 index 0000000000..72a6db45a1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -0,0 +1,236 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus-node-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus-node-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-node-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-node-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-node-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-node-exporter.name" . }} +{{ include "prometheus-node-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-node-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-node-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-node-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-node-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-node-exporter.image" -}} +{{- $temp_registry := (include "system_default_registry" .) }} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if $temp_registry }} +{{- printf "%s%s:%s@%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if $temp_registry }} +{{- printf "%s%s:%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-node-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-node-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-node-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-node-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* Sets sidecar volumeMounts */}} +{{- define "prometheus-node-exporter.sidecarVolumeMounts" -}} +{{- range $_, $mount := $.Values.sidecarVolumeMount }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- end }} +{{- range $_, $mount := $.Values.sidecarHostVolumeMounts }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- if $mount.mountPropagation }} + mountPropagation: {{ $mount.mountPropagation }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml new file mode 100644 index 0000000000..c256dba73d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: + {{- if $.Values.kubeRBACProxy.enabled }} + - apiGroups: [ "authentication.k8s.io" ] + resources: + - tokenreviews + verbs: [ "create" ] + - apiGroups: [ "authorization.k8s.io" ] + resources: + - subjectaccessreviews + verbs: [ "create" ] + {{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..653305ad9e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + name: {{ template "prometheus-node-exporter.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml new file mode 100644 index 0000000000..48d274f1b1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -0,0 +1,309 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ ternary true false (or .Values.serviceAccount.automountServiceAccountToken .Values.kubeRBACProxy.enabled) }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.extraInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-node-exporter.serviceAccountName" . }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + containers: + {{- $servicePort := ternary .Values.kubeRBACProxy.port .Values.service.port .Values.kubeRBACProxy.enabled }} + - name: node-exporter + image: {{ include "prometheus-node-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + {{- if .Values.hostRootFsMount.enabled }} + - --path.rootfs=/host/root + {{- if semverCompare ">=1.4.0-0" (coalesce .Values.version .Values.image.tag .Chart.AppVersion) }} + - --path.udev.data=/host/root/run/udev/data + {{- end }} + {{- end }} + - --web.listen-address=[$(HOST_IP)]:{{ $servicePort }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: HOST_IP + {{- if .Values.kubeRBACProxy.enabled }} + value: 127.0.0.1 + {{- else if .Values.service.listenOnAllInterfaces }} + value: 0.0.0.0 + {{- else }} + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + {{- end }} + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - name: {{ .Values.service.portName }} + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + volumeMounts: + - name: proc + mountPath: /host/proc + {{- with .Values.hostProcFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + - name: sys + mountPath: /host/sys + {{- with .Values.hostSysFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- if .Values.hostRootFsMount.enabled }} + - name: root + mountPath: /host/root + {{- with .Values.hostRootFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- with $mount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- range .Values.sidecars }} + {{- $overwrites := dict "volumeMounts" (concat (include "prometheus-node-exporter.sidecarVolumeMounts" $ | fromYamlArray) (.volumeMounts | default list) | default list) }} + {{- $defaults := dict "image" (include "prometheus-node-exporter.image" $) "securityContext" $.Values.containerSecurityContext "imagePullPolicy" $.Values.image.pullPolicy }} + - {{- toYaml (merge $overwrites . $defaults) | nindent 10 }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 12 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- if .Values.kubeRBACProxy.image.sha }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}@sha256:{{ .Values.kubeRBACProxy.image.sha }}" + {{- else }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}" + {{- end }} + ports: + - containerPort: {{ .Values.service.port}} + name: {{ .Values.kubeRBACProxy.portName }} + {{- if .Values.kubeRBACProxy.enableHostPort }} + hostPort: {{ .Values.service.port }} + {{- end }} + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: + {{- toYaml .Values.kubeRBACProxy.resources | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + {{- with .Values.kubeRBACProxy.env }} + env: + {{- range $key, $value := $.Values.kubeRBACProxy.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: + {{ toYaml .Values.kubeRBACProxy.containerSecurityContext | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.restartPolicy }} + restartPolicy: {{ . }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + {{- if .Values.hostRootFsMount.enabled }} + - name: root + hostPath: + path: / + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- with $mount.type }} + type: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml new file mode 100644 index 0000000000..56b695203a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml @@ -0,0 +1,18 @@ +{{- if .Values.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +subsets: + - addresses: + {{- range .Values.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml new file mode 100644 index 0000000000..2b21b71062 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl . $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml new file mode 100644 index 0000000000..825722729d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml @@ -0,0 +1,23 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingress: + - ports: + - port: {{ .Values.service.port }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml new file mode 100644 index 0000000000..f88da6a34e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.podmonitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-node-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..ee5bbba4a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml @@ -0,0 +1,14 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ include "prometheus-node-exporter.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..160f2bbf7a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ include "prometheus-node-exporter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml new file mode 100644 index 0000000000..f3b52e1120 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml @@ -0,0 +1,49 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.rbac.pspAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + - 'hostPath' + hostNetwork: true + hostIPC: false + hostPID: true + hostPorts: + - min: 0 + max: 65535 + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml new file mode 100644 index 0000000000..814e110337 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + namespace: {{ include "prometheus-node-exporter.namespace" . }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "prometheus-node-exporter.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "prometheus-node-exporter.fullname" . }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/service.yaml new file mode 100644 index 0000000000..a065e46e39 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/service.yaml @@ -0,0 +1,29 @@ +{{- if .Values.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.service.ipDualStack.ipFamilyPolicy }} +{{- end }} + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: {{ .Values.service.portName }} + selector: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..5c3348c09b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..6d6e440473 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.monitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + metricRelabelings: + {{- with .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..2c2705f872 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: node-exporter + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ . }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: {{ include "prometheus-node-exporter.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/values.yaml new file mode 100644 index 0000000000..b9f2f7ab87 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/prometheus-node-exporter/values.yaml @@ -0,0 +1,530 @@ +# Default values for prometheus-node-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-node-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: v1.7.0 + pullPolicy: IfNotPresent + digest: "" + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +global: + cattle: + psp: + enable: true + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "docker.io" + +# Configure kube-rbac-proxy. When enabled, creates a kube-rbac-proxy to protect the node-exporter http endpoint. +# The requests are served through the same service but requests are HTTPS. +kubeRBACProxy: + enabled: false + ## Set environment variables as name/value pairs + env: {} + # VARIABLE: value + image: + registry: docker.io + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.15.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-proxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: {} + + # Specify the port used for the Node exporter container (upstream port) + port: 8100 + # Specify the name of the container port + portName: http + # Configure a hostPort. If true, hostPort will be enabled in the container and set to service.port. + enableHostPort: false + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +service: + enabled: true + type: ClusterIP + port: 9796 + targetPort: 9796 + nodePort: + portName: metrics + listenOnAllInterfaces: true + annotations: + prometheus.io/scrape: "true" + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + +# Set a NetworkPolicy with: +# ingress only on service.port +# no egress permitted +networkPolicy: + enabled: false + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + + jobLabel: "" + + # List of pod labels to add to node exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: [] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Node Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +# Specify the container restart policy passed to the Node Export container +# Possible Values: Always (default)|OnFailure|Never +restartPolicy: null + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + +containerSecurityContext: + readOnlyRootFilesystem: true + # capabilities: + # add: + # - SYS_TIME + +rbac: + ## If true, create & use RBAC resources + ## + create: true + pspAnnotations: {} + +# for deployments that have node_exporter deployed outside of the cluster, list +# their addresses here +endpoints: [] + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +# Mount the node's root file system (/) at /host/root in the container +hostRootFsMount: + enabled: true + # Defines how new mounts in existing mounts on the node or in the container + # are propagated to the container or node, respectively. Possible values are + # None, HostToContainer, and Bidirectional. If this field is omitted, then + # None is used. More information on: + # https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation + mountPropagation: HostToContainer + +# Mount the node's proc file system (/proc) at /host/proc in the container +hostProcFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +# Mount the node's sys file system (/sys) at /host/sys in the container +hostSysFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to node exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to node exporter pods +podLabels: {} + +# Annotations to be added to node exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-node-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: linux + # kubernetes.io/arch: amd64 + +# Specify grace period for graceful termination of pods. Defaults to 30 if null or not specified +terminationGracePeriodSeconds: null + +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + +# Enable or disable container termination message settings +# https://kubernetes.io/docs/tasks/debug/debug-application/determine-reason-pod-failure/ +terminationMessageParams: + enabled: false + # If enabled, specify the path for termination messages + terminationMessagePath: /dev/termination-log + # If enabled, specify the policy for termination messages + terminationMessagePolicy: File + + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.diskstats.ignored-devices=^(ram|loop|fd|(h|s|v)d[a-z]|nvme\\d+n\\d+p)\\d+$ +# - --collector.textfile.directory=/run/prometheus + +## Additional mounts from the host to node-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# https://kubernetes.io/docs/concepts/storage/volumes/#hostpath-volume-types +# type: "" (Default)|DirectoryOrCreate|Directory|FileOrCreate|File|Socket|CharDevice|BlockDevice +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file; fields image,imagePullPolicy,securityContext take default value from main container +## +sidecars: [] +# - name: nvidia-dcgm-exporter +# image: nvidia/dcgm-exporter:1.4.3 +# volumeMounts: +# - name: tmp +# mountPath: /tmp + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +# - name: collector-textfiles +# mountPath: /run/prometheus +# readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +# Enable vertical pod autoscaler support for prometheus-node-exporter +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# Extra manifests to deploy as an array +extraManifests: [] + # - | + # apiVersion: v1 + # kind: ConfigMap + # metadata: + # name: prometheus-extra + # data: + # extra-data: "value" + +# Override version of app, required if image.tag is defined and does not follow semver +version: "" diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/Chart.yaml new file mode 100644 index 0000000000..dbf4d0b815 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2ControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2ControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/Chart.yaml new file mode 100644 index 0000000000..87495a6a6b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Etcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Etcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/Chart.yaml new file mode 100644 index 0000000000..4bce6d41ea --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2IngressNginx +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2IngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/Chart.yaml new file mode 100644 index 0000000000..0a316e05c7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Proxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Proxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/Chart.yaml new file mode 100644 index 0000000000..fa6aa5ac66 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Scheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rke2Scheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/Chart.yaml new file mode 100644 index 0000000000..df00a46b66 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/Chart.yaml new file mode 100644 index 0000000000..96b33f0bc8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeEtcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/Chart.yaml new file mode 100644 index 0000000000..bc49bcefd2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeIngressNginx +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeIngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/Chart.yaml new file mode 100644 index 0000000000..18eac324d4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeProxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/Chart.yaml new file mode 100644 index 0000000000..8c53b63bcc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeScheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/rkeScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/Chart.yaml new file mode 100644 index 0000000000..784bb0ec7e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +appVersion: 0.25.1 +description: A Helm chart for prometheus windows-exporter +home: https://github.com/prometheus-community/windows_exporter/ +keywords: +- windows-exporter +- windows +- prometheus +- exporter +maintainers: +- email: github@jkroepke.de + name: jkroepke +name: windowsExporter +sources: +- https://github.com/prometheus-community/windows_exporter/ +type: application +version: 0.3.1 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/README.md new file mode 100644 index 0000000000..1da1c64e12 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/README.md @@ -0,0 +1,42 @@ +# Prometheus `Windows Exporter` + +Prometheus exporter for hardware and OS metrics exposed by Windows kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a prometheus [`Windows Exporter`](http://github.com/prometheus-community/windows_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-windows-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-windows-exporter +``` diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 new file mode 100644 index 0000000000..9cbed7112d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 @@ -0,0 +1,31 @@ +$ErrorActionPreference = 'Continue' + +function CheckFirewallRuleError { + # We hit an error. This can happen for a number of reasons, including if the rule already exists + if ($error[0]) { + if (($error[0].Exception.NativeErrorCode) -and ($error[0].Exception.NativeErrorCode.ToString() -eq "AlreadyExists")) { + # Previous versions of monitoring may have already created this Firewall Rule + # Because of this, if the rule alreadys exists there is no need to delete and recreate it. + Write-Host "Detected Existing Firewall Rule, Nothing To Do" + } else { + Write-Host "Error Encountered Setting Up Required Firewall Rule" + $error[0].Exception + exit 1 + } + } +} + +Write-Host "Attempting To Configure Firewall Rules For Ports 9796, 10250" + +# This is the exact same firewall rule that has historically been created by rancher-wins +# https://github.com/rancher/wins/blob/91f670c47f19c6d9fe97d8f66a695d3081ad994f/pkg/apis/process_service_mgmt.go#L149 +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-9796 -Name rancher-wins-windows-exporter-TCP-9796 -Action Allow -Protocol TCP -LocalPort 9796 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Node Exporter Firewall Rule Successfully Created" + +# This rule is required in order to have the Rancher UI display node metrics in the 'Nodes' tab of the cluster explorer +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-10250 -Name rancher-wins-windows-exporter-TCP-10250 -Action Allow -Protocol TCP -LocalPort 10250 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Prometheus Metrics Firewall Rule Successfully Created" + +Write-Host "All Firewall Rules Successfully Configured" diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..c9a5d6db8c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/_helpers.tpl @@ -0,0 +1,216 @@ +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "prometheus-windows-exporter.fullname" -}} +{{ printf "%s-windows-exporter" .Release.Name }} +{{- end -}} + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "windowsExporter.renamedMetricsRelabeling" -}} +{{- range $original, $new := (include "windowsExporter.renamedMetrics" . | fromJson) -}} +- sourceLabels: [__name__] + regex: {{ $original }} + replacement: '{{ $new }}' + targetLabel: __name__ +{{ end -}} +{{- end -}} + +{{- define "windowsExporter.labels" -}} +k8s-app: {{ template "prometheus-windows-exporter.fullname" . }} +release: {{ .Release.Name }} +component: "windows-exporter" +provider: kubernetes +{{- end -}} + +{{- define "windowsExporter.renamedMetrics" -}} +{{- $renamed := dict -}} +{{/* v0.15.0 */}} +{{- $_ := set $renamed "windows_mssql_transactions_active_total" "windows_mssql_transactions_active" -}} +{{/* v0.16.0 */}} +{{- $_ := set $renamed "windows_adfs_ad_login_connection_failures" "windows_adfs_ad_login_connection_failures_total" -}} +{{- $_ := set $renamed "windows_adfs_certificate_authentications" "windows_adfs_certificate_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_device_authentications" "windows_adfs_device_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_extranet_account_lockouts" "windows_adfs_extranet_account_lockouts_total" -}} +{{- $_ := set $renamed "windows_adfs_federated_authentications" "windows_adfs_federated_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_passport_authentications" "windows_adfs_passport_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_failed" "windows_adfs_password_change_failed_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_succeeded" "windows_adfs_password_change_succeeded_total" -}} +{{- $_ := set $renamed "windows_adfs_token_requests" "windows_adfs_token_requests_total" -}} +{{- $_ := set $renamed "windows_adfs_windows_integrated_authentications" "windows_adfs_windows_integrated_authentications_total" -}} +{{- $_ := set $renamed "windows_net_packets_outbound_errors" "windows_net_packets_outbound_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_discarded" "windows_net_packets_received_discarded_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_errors" "windows_net_packets_received_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_total" "windows_net_packets_received_total_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_unknown" "windows_net_packets_received_unknown_total" -}} +{{- $_ := set $renamed "windows_dns_memory_used_bytes_total" "windows_dns_memory_used_bytes" -}} +{{- $renamed | toJson -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-windows-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-windows-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-windows-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-windows-exporter.name" . }} +{{ include "prometheus-windows-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-windows-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-windows-exporter.fullname" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-windows-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-windows-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-windows-exporter.image" -}} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-windows-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-windows-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-windows-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-windows-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/config.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/config.yaml new file mode 100644 index 0000000000..25f1fa69c2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + config.yml: | + {{- .Values.config | nindent 4 }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/daemonset.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/daemonset.yaml new file mode 100644 index 0000000000..be7feb3ed1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/daemonset.yaml @@ -0,0 +1,200 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "windowsExporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + initContainers: + - name: configure-firewall + image: {{ include "prometheus-windows-exporter.image" . }} + command: + - C:\WINDOWS\System32\WindowsPowerShell\v1.0\powershell.exe + args: ["-f", "scripts/configure-firewall.ps1"] + volumeMounts: + - mountPath: /scripts + name: exporter-scripts + {{- with .Values.extraInitContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-windows-exporter.fullname" . }} + containers: + - name: windows-exporter + image: {{ include "prometheus-windows-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --config.file=%CONTAINER_SANDBOX_MOUNT_POINT%/config.yml + - --collector.textfile.directories=%CONTAINER_SANDBOX_MOUNT_POINT% + - --web.listen-address=:{{ .Values.service.port }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + hostPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /config.yml + subPath: config.yml + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- if or .Values.sidecarVolumeMount .Values.sidecarHostVolumeMounts }} + volumeMounts: + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: exporter-scripts + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + - name: config + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/podmonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/podmonitor.yaml new file mode 100644 index 0000000000..bbb6c39340 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.podmonitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-windows-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-windows-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml new file mode 100644 index 0000000000..f514c8161a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{ (.Files.Glob "scripts/*").AsConfig | indent 2 }} + diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/service.yaml new file mode 100644 index 0000000000..267b796f63 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/service.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- if or .Values.prometheus.monitor.enabled .Values.prometheus.podMonitor.enabled }} + {{- with .Values.service.annotations }} + annotations: + {{- unset . "prometheus.io/scrape" | toYaml | nindent 4 }} + {{- end }} + {{- else }} + annotations: + prometheus.io/scrape: "true" + {{- with .Values.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port }} + protocol: TCP + appProtocol: http + name: {{ .Values.service.portName }} + selector: + {{- include "windowsExporter.labels" . | nindent 4 }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..14c1c46807 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-windows-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..2effc07758 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.monitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + metricRelabelings: +{{- include "windowsExporter.renamedMetricsRelabeling" . | nindent 6 -}} + - sourceLabels: [__name__] + regex: 'wmi_(.*)' + replacement: 'windows_$1' + targetLabel: __name__ + - sourceLabels: [volume, nic] + regex: (.*);(.*) + separator: '' + targetLabel: device + action: replace + replacement: $1$2 + - sourceLabels: [__name__] + regex: windows_cs_logical_processors + replacement: 'system' + targetLabel: mode + relabelings: + - separator: ':' + sourceLabels: + - __meta_kubernetes_pod_host_ip + - __meta_kubernetes_pod_container_port_number + targetLabel: instance +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/values.yaml new file mode 100644 index 0000000000..04569505d6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/charts/windowsExporter/values.yaml @@ -0,0 +1,366 @@ +# Default values for prometheus-windows-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-windows-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: "0.25.1" + pullPolicy: IfNotPresent + digest: "" + +config: |- + collectors: + enabled: '[defaults],tcp,memory,container' + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +global: + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + cattle: + systemDefaultRegistry: "" + +service: + type: ClusterIP + port: 9796 + nodePort: + portName: windows-metrics + annotations: {} + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: true + additionalLabels: {} + namespace: "" + + jobLabel: "component" + + # List of pod labels to add to windows exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: ["component"] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Windows Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m +# memory: 30Mi + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + +rbac: + ## If true, create & use RBAC resources + ## + create: true + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to windows exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to windows exporter pods +podLabels: {} + +# Annotations to be added to windows exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-windows-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: windows + # kubernetes.io/arch: amd64 + +tolerations: + - effect: NoSchedule + operator: Exists + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.service.services-where +# - "Name LIKE 'sql%'" + +## Additional mounts from the host to windows-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file +## +sidecars: [] +## - name: nvidia-dcgm-exporter +## image: nvidia/dcgm-exporter:1.4.3 + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +## - name: collector-textfiles +## mountPath: /run/prometheus +## readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/ingress-nginx/nginx.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/ingress-nginx/nginx.json new file mode 100644 index 0000000000..565352235a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/ingress-nginx/nginx.json @@ -0,0 +1,1445 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "$datasource", + "enable": true, + "expr": "sum(changes(nginx_ingress_controller_config_last_reload_successful_timestamp_seconds{instance!=\"unknown\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[30s])) by (controller_class)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Config Reloads", + "showIn": 0, + "step": "30s", + "tagKeys": "controller_class", + "tags": [], + "titleFormat": "Config Reloaded", + "type": "tags" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1534359654832, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 82, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",state=\"active\"}[2m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Connections", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Controller Success Rate (non-4|5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 81, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(irate(nginx_ingress_controller_success{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[1m])) * 60", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Config Reloads", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(nginx_ingress_controller_config_last_reload_successful{controller_pod=~\"$controller\",controller_namespace=~\"$namespace\"} == 0)", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Last Config Failed", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "200px", + "id": 86, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "network", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Request Volume", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Success Rate (non-4|5xx responses)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 77, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) ", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "", + "id": 79, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg (rate (nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) ", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "hideTimeOverride": false, + "id": 75, + "links": [], + "pageSize": 7, + "repeat": null, + "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Ingress", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ingress", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ops" + }, + { + "alias": "Errors", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "IN", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "OUT", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "Bps" + } + ], + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "E" + }, + { + "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "F" + }, + { + "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "G" + } + ], + "timeFrom": null, + "title": "Ingress Percentile Response Times and Transfer Rates", + "transform": "table", + "transparent": false, + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "height": "1024", + "id": 85, + "links": [], + "pageSize": 7, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "TTL", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0", + "691200" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "avg(nginx_ingress_controller_ssl_expire_time_seconds{kubernetes_pod_name=~\"$controller\",namespace=~\"$namespace\",ingress=~\"$ingress\"}) by (host) - time()", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ host }}", + "metric": "gke_letsencrypt_cert_expiration", + "refId": "A", + "step": 1 + } + ], + "title": "Ingress Certificate Expiry", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller Class", + "multi": false, + "name": "controller_class", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller", + "multi": false, + "name": "controller", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller_pod=~\"$controller\"}, ingress) ", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Ingress Controller", + "uid": "nginx", + "version": 1 +} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/ingress-nginx/request-handling-performance.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/ingress-nginx/request-handling-performance.json new file mode 100644 index 0000000000..156e33123d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/ingress-nginx/request-handling-performance.json @@ -0,0 +1,963 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": 9614, + "graphTooltip": 1, + "id": null, + "iteration": 1582146566338, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total time taken for nginx and upstream servers to process a request and send a response", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total request handling time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The time spent on receiving the response from the upstream server", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 94, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum by (path)(\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, its median upstream response time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n .5,\n sum by (le, path)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median upstream response time by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Percentage of 4xx and 5xx responses among all responses.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~ \"[4-5].*\"\n}[1m])) / sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error rate by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, the sum of upstream request time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_response_duration_seconds_sum{ingress =~ \"$ingress\"}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream time consumed by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum (\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~\"[4-5].*\",\n }[1m]\n )\n ) by(path, status)\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }} {{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate (\n nginx_ingress_controller_response_size_sum {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path) / sum (\n rate(\n nginx_ingress_controller_response_size_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "D" + }, + { + "expr": " sum (rate(nginx_ingress_controller_response_size_bucket{\n ingress =~ \"$ingress\",\n }[1m])) by (le)\n", + "hide": true, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average response size by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_sum {\n ingress =~ \"$ingress\",\n }[1m]\n)) / sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "average", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream service latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 22, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(nginx_ingress_controller_requests, ingress) ", + "hide": 0, + "includeAll": true, + "label": "Service Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests, ingress) ", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Request Handling Performance", + "uid": "4GFbkOsZk", + "version": 1 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json new file mode 100644 index 0000000000..1d4943501b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json @@ -0,0 +1,793 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m] ({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m])) by (instance)", + "interval": "", + "legendFormat": "Load[5m] ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m])) by (instance)", + "interval": "", + "legendFormat": "Load[1m] ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m])) by (instance)", + "interval": "", + "legendFormat": "Load[15m] ({{instance}})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) by (instance) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes) by (instance) ", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Read ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Write ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Errors ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Errors ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Dropped ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Dropped ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster (Nodes)", + "uid": "rancher-cluster-nodes-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/cluster/rancher-cluster.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/cluster/rancher-cluster.json new file mode 100644 index 0000000000..24385a237a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/cluster/rancher-cluster.json @@ -0,0 +1,776 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval]))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes)", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster", + "uid": "rancher-cluster-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/bundle.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/bundle.json new file mode 100644 index 0000000000..698f48aeed --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/bundle.json @@ -0,0 +1,246 @@ +{ + "description": "Bundle", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Bundle", + "uid": "fleet-bundle" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/bundledeployment.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/bundledeployment.json new file mode 100644 index 0000000000..c81f7a6212 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/bundledeployment.json @@ -0,0 +1,219 @@ +{ + "description": "BundleDeployment", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"}) / sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\"})" + } + ], + "title": "Ready BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundledeployment_state, cluster_namespace)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / BundleDeployment", + "uid": "fleet-bundledeployment" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/cluster.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/cluster.json new file mode 100644 index 0000000000..73bdea4834 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/cluster.json @@ -0,0 +1,484 @@ +{ + "description": "Cluster", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"}) / sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_desired_ready_git_repos, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_desired_ready_git_repos{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Cluster", + "uid": "fleet-cluster" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/clustergroup.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/clustergroup.json new file mode 100644 index 0000000000..ce3df87b21 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/clustergroup.json @@ -0,0 +1,468 @@ +{ + "description": "ClusterGroup", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"}) - sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})) / sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_group_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_group_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / ClusterGroup", + "uid": "fleet-cluster-group" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/controller-runtime.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/controller-runtime.json new file mode 100644 index 0000000000..23a81f2a8c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/controller-runtime.json @@ -0,0 +1,454 @@ +{ + "description": "Controller Runtime", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "controller_runtime_active_workers{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{controller}} {{instance}}" + } + ], + "title": "Number of Workers in Use", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Reconciliation Error Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Total Reconciliation Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "workqueue_depth{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "WorkQueue Depth", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P50", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P90", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P99", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_adds_total{job=\"$job\", namespace=\"$namespace\"}[2m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Add Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(workqueue_unfinished_work_seconds{job=\"$job\", namespace=\"$namespace\"}[5m])", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Unfinished Seconds", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 10, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 50th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 11, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 90th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 12, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 99th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 96 + }, + "id": 13, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_retries_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Retries Rate", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(controller_runtime_reconcile_total, namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "job", + "query": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"}, job)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Controller-Runtime", + "uid": "fleet-controller-runtime" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/gitrepo.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/gitrepo.json new file mode 100644 index 0000000000..1a50c2937d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/fleet/gitrepo.json @@ -0,0 +1,325 @@ +{ + "description": "GitRepo", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_gitrepo_desired_ready_clusters, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_gitrepo_desired_ready_clusters{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / GitRepo", + "uid": "fleet-gitrepo" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/home/rancher-default-home.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/home/rancher-default-home.json new file mode 100644 index 0000000000..3fce207561 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/home/rancher-default-home.json @@ -0,0 +1,1290 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "", + "type": "welcome" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[5m])))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "CPU Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 4 + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"})) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Memory Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 4 + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Disk Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 9 + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode!=\"idle\"}[5m]))", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 9 + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_cpu_cores{}) OR sum(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 9 + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 9 + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_memory_bytes{}) OR sum(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 9 + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) - sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 2051, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", mode=\"idle\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 2052, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "100 * (1- sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) by (instance) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) by (instance))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 2053, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(1 - ((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"} OR on() vector(0)))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0)))) * 100", + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "(1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "B" + }, + { + "expr": "(1 - (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) / sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "folderId": 0, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 18 + }, + "headings": true, + "id": 3, + "limit": 30, + "links": [], + "query": "", + "recent": true, + "search": true, + "starred": false, + "tags": [], + "title": "Dashboards", + "type": "dashlist" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 2055, + "options": { + "content": "## About Rancher Monitoring\n\nRancher Monitoring is a Helm chart developed by Rancher that is powered by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). It is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm chart maintained by the Prometheus community.\n\nBy default, the chart deploys Grafana alongside a set of Grafana dashboards curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project.\n\nFor more information on how Rancher Monitoring differs from [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack), please view the CHANGELOG.md of the rancher-monitoring chart located in the [rancher/charts](https://github.com/rancher/charts) repository.\n\nFor more information about how to configure Rancher Monitoring, please view the [Rancher docs](https://rancher.com/docs/rancher/v2.x/en/).\n\n", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "text" + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Home", + "uid": "rancher-home-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json new file mode 100644 index 0000000000..8af4b81ce0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json @@ -0,0 +1,687 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 32, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic In ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic Out ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes) by (instance)", + "interval": "", + "legendFormat": "DB Size ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Watch Streams ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Lease Watch Stream ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Committed ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Applied ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Failed ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending) by (instance)", + "interval": "", + "legendFormat": "Proposal Pending ({{instance}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Rate ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Failure Rate ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync ({{instance}})", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd (Nodes)", + "uid": "rancher-etcd-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-etcd.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-etcd.json new file mode 100644 index 0000000000..0c058cafb9 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-etcd.json @@ -0,0 +1,669 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 33, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic In", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic Out", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes)", + "interval": "", + "legendFormat": "DB Size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Watch Streams", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Lease Watch Stream", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Committed", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Applied", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Failed", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending)", + "interval": "", + "legendFormat": "Proposal Pending", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Rate", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Failure Rate", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd", + "uid": "rancher-etcd-1", + "version": 4 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json new file mode 100644 index 0000000000..b31358eaaf --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json @@ -0,0 +1,527 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 30, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (instance, code)", + "interval": "", + "legendFormat": "{{code}}({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (instance, name)", + "interval": "", + "legendFormat": "Deployment Depth ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (instance, name)", + "interval": "", + "legendFormat": "Volumes Depth ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicaSet Depth ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (instance, name)", + "interval": "", + "legendFormat": "Service Depth ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (instance, name)", + "interval": "", + "legendFormat": "ServiceAccount Depth ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (instance, name)", + "interval": "", + "legendFormat": "Endpoint Depth ({{instance}})", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (instance, name)", + "interval": "", + "legendFormat": "DaemonSet Depth ({{instance}})", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (instance, name)", + "interval": "", + "legendFormat": "StatefulSet Depth ({{instance}})", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicationManager Depth ({{instance}})", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"}) by (instance)", + "interval": "", + "legendFormat": "Reading ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"}) by (instance)", + "interval": "", + "legendFormat": "Waiting ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"}) by (instance)", + "interval": "", + "legendFormat": "Writing ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Accepted ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Handled ({{instance}})", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components (Nodes)", + "uid": "rancher-k8s-components-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-k8s-components.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-k8s-components.json new file mode 100644 index 0000000000..44cf97f9fd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/k8s/rancher-k8s-components.json @@ -0,0 +1,519 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 31, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (code)", + "interval": "", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (name)", + "interval": "", + "legendFormat": "Deployment Depth", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (name)", + "interval": "", + "legendFormat": "Volumes Depth", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (name)", + "interval": "", + "legendFormat": "Replicaset Depth", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (name)", + "interval": "", + "legendFormat": "Service Depth", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (name)", + "interval": "", + "legendFormat": "ServiceAccount Depth", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (name)", + "interval": "", + "legendFormat": "Endpoint Depth", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (name)", + "interval": "", + "legendFormat": "DaemonSet Depth", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (name)", + "interval": "", + "legendFormat": "StatefulSet Depth", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (name)", + "interval": "", + "legendFormat": "ReplicationManager Depth", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"})", + "interval": "", + "legendFormat": "Reading", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"})", + "interval": "", + "legendFormat": "Waiting", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"})", + "interval": "", + "legendFormat": "Writing", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Accepted", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Handled", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components", + "uid": "rancher-k8s-components-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/nodes/rancher-node-detail.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/nodes/rancher-node-detail.json new file mode 100644 index 0000000000..920fb94cf7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/nodes/rancher-node-detail.json @@ -0,0 +1,805 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{mode}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\"}[$__rate_interval])) by (mode)", + "interval": "", + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / (node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device))", + "interval": "", + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Read ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Write ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Errors ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Errors ({{device}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Dropped ({{device}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Dropped ({{device}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node (Detail)", + "uid": "rancher-node-detail-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/nodes/rancher-node.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/nodes/rancher-node.json new file mode 100644 index 0000000000..367df3cc9d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/nodes/rancher-node.json @@ -0,0 +1,792 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\", mode=\"idle\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / sum(node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node", + "uid": "rancher-node-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/performance/performance-debugging.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/performance/performance-debugging.json new file mode 100644 index 0000000000..454bc39390 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/performance/performance-debugging.json @@ -0,0 +1,1652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_sum[5m]))\n/\nsum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_count[5m])))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Average Execution Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1390", + "format": "short", + "label": "Execution Time in Seconds", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1391", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(steve_api_request_time_sum{resource!=\"subscribe\"}[5m]))\n/\nsum by (resource, method, code) (rate(steve_api_request_time_count{resource!=\"subscribe\"}[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Rancher API Average Request Times Over Last 5 Minutes (Top 20) (Subscribes Omitted)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:178", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:179", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "rate(steve_api_request_time_sum{resource=\"subscribe\"}[5m])\n/\nrate(steve_api_request_time_count{resource=\"subscribe\"}[5m])", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Subscribe Average Request Times Over Last 5 Minutes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:368", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:369", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,workqueue_depth)", + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Controller Work Queue Depth (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1553", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1554", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 16, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method, code) (steve_api_total_requests))", + "instant": false, + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Rancher Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:290", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:291", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method) (steve_api_total_requests{code!=\"200\",code!=\"201\"}))", + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Failed Rancher API Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:428", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:429", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_store_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_store_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Store Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:662", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:663", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 62 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_client_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_client_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Client Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1710", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1711", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,lasso_controller_total_cached_object)", + "interval": "", + "legendFormat": "{{kind}} {{version}} {{group}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cached Objects by GroupVersionKind (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:744", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:745", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 78 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Handler Executions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:824", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:825", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 86 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20, sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution{has_error=\"true\"}\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Handler Executions with Error (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1230", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1231", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 102 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution{has_error=\"true\"}[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 110 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,session_server_total_transmit_bytes)", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Data Transmitted by Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1953", + "format": "decbytes", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1954", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 118 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "session_server_total_transmit_error_bytes", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Errors for Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2045", + "format": "ms", + "label": "Error Data", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2046", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 126 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "session_server_total_add_websocket_session - (session_server_total_remove_websocket_session or (0 * session_server_total_add_websocket_session))", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Active Connections (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 134 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_remove_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Removed Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 142 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_add_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Added Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2117", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2118", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rancher Performance Debugging", + "uid": "tfrfU0a7k", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/pods/rancher-pod-containers.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/pods/rancher-pod-containers.json new file mode 100644 index 0000000000..cf78a2204c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/pods/rancher-pod-containers.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "CFS throttled ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "System ({{container}})", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Total ({{container}})", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "User ({{container}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}) by (container)", + "interval": "", + "legendFormat": "({{container}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Dropped ({{container}})", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Errors ({{container}})", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Errors ({{container}})", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Dropped ({{container}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Write ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Read ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod (Containers)", + "uid": "rancher-pod-containers-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/pods/rancher-pod.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/pods/rancher-pod.json new file mode 100644 index 0000000000..4859eccc74 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/pods/rancher-pod.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod", + "uid": "rancher-pod-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/workloads/rancher-workload-pods.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/workloads/rancher-workload-pods.json new file mode 100644 index 0000000000..92c0d24a6e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/workloads/rancher-workload-pods.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "CFS throttled ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "System ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Total ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "User ({{pod}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Dropped ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Errors ({{pod}})", + "refId": "D" + }, + { + "expr": "(sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Errors ({{pod}})", + "refId": "E" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Dropped ({{pod}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Write ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Read ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload (Pods)", + "uid": "rancher-workload-pods-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/workloads/rancher-workload.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/workloads/rancher-workload.json new file mode 100644 index 0000000000..9f5317c2f0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/rancher/workloads/rancher-workload.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum((sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum((sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(container_memory_working_set_bytes{namespace=~\"$namespace\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum((sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum((sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum((sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload", + "uid": "rancher-workload-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh new file mode 100644 index 0000000000..89431e7132 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e +set -x + +# node-exporter +kubectl delete daemonset -l app=prometheus-node-exporter,release=rancher-monitoring --ignore-not-found=true + +# prometheus-adapter +kubectl delete deployments -l app=prometheus-adapter,release=rancher-monitoring --ignore-not-found=true + +# kube-state-metrics +kubectl delete deployments -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true +kubectl delete statefulsets -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/NOTES.txt new file mode 100644 index 0000000000..371f3ae398 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "kube-prometheus-stack.namespace" . }} get pods -l "release={{ $.Release.Name }}" + +Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/_helpers.tpl new file mode 100644 index 0000000000..d2207dd90d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/_helpers.tpl @@ -0,0 +1,459 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +{{/* +https://github.com/helm/helm/issues/4535#issuecomment-477778391 +Usage: {{ include "call-nested" (list . "SUBCHART_NAME" "TEMPLATE") }} +e.g. {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +*/}} +{{- define "call-nested" }} +{{- $dot := index . 0 }} +{{- $subchart := index . 1 | splitList "." }} +{{- $template := index . 2 }} +{{- $values := $dot.Values }} +{{- range $subchart }} +{{- $values = index $values . }} +{{- end }} +{{- include $template (dict "Chart" (dict "Name" (last $subchart)) "Values" $values "Release" $dot.Release "Capabilities" $dot.Capabilities) }} +{{- end }} + +# Special Exporters +{{- define "exporter.kubeEtcd.enabled" -}} +{{- if or .Values.kubeEtcd.enabled .Values.rkeEtcd.enabled .Values.kubeAdmEtcd.enabled .Values.rke2Etcd.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.enabled" -}} +{{- if or .Values.kubeControllerManager.enabled .Values.rkeControllerManager.enabled .Values.k3sServer.enabled .Values.kubeAdmControllerManager.enabled .Values.rke2ControllerManager.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.enabled" -}} +{{- if or .Values.kubeScheduler.enabled .Values.rkeScheduler.enabled .Values.k3sServer.enabled .Values.kubeAdmScheduler.enabled .Values.rke2Scheduler.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.enabled" -}} +{{- if or .Values.kubeProxy.enabled .Values.rkeProxy.enabled .Values.k3sServer.enabled .Values.kubeAdmProxy.enabled .Values.rke2Proxy.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.enabled" -}} +{{- if or .Values.kubelet.enabled .Values.hardenedKubelet.enabled .Values.k3sServer.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-controller-manager +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-scheduler +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-proxy +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kubelet +{{- end -}} +{{- end }} + +{{- define "kubelet.serviceMonitor.resourcePath" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if not (eq .Values.kubelet.serviceMonitor.resourcePath "/metrics/resource/v1alpha1") -}} +{{ .Values.kubelet.serviceMonitor.resourcePath }} +{{- else if semverCompare ">=1.20.0-0" $kubeTargetVersion -}} +/metrics/resource +{{- else -}} +/metrics/resource/v1alpha1 +{{- end -}} +{{- end }} + +{{- define "rancher.serviceMonitor.selector" -}} +{{- if .Values.rancherMonitoring.selector }} +{{ .Values.rancherMonitoring.selector | toYaml }} +{{- else }} +{{- $rancherDeployment := (lookup "apps/v1" "Deployment" "cattle-system" "rancher") }} +{{- if $rancherDeployment }} +matchLabels: + app: rancher + chart: {{ index $rancherDeployment.metadata.labels "chart" }} + release: rancher +{{- end }} +{{- end }} +{{- end }} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Prometheus Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "kube-prometheus-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "kube-prometheus-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 26 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Fullname suffixed with -operator */}} +{{/* Adding 9 to 26 truncation of kube-prometheus-stack.fullname */}} +{{- define "kube-prometheus-stack.operator.fullname" -}} +{{- if .Values.prometheusOperator.fullnameOverride -}} +{{- .Values.prometheusOperator.fullnameOverride | trunc 35 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-operator" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} +{{- end }} + +{{/* Prometheus custom resource instance name */}} +{{- define "kube-prometheus-stack.prometheus.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-prometheus" }} +{{- end }} +{{- end }} + +{{/* Prometheus apiVersion for networkpolicy */}} +{{- define "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} + +{{/* Alertmanager custom resource instance name */}} +{{- define "kube-prometheus-stack.alertmanager.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-alertmanager" -}} +{{- end }} +{{- end }} + +{{/* Fullname suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.fullname" -}} +{{- printf "%s-thanos-ruler" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} + +{{/* Shortened name suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.name" -}} +{{- default (printf "%s-thanos-ruler" (include "kube-prometheus-stack.name" .)) .Values.thanosRuler.name -}} +{{- end }} + + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "kube-prometheus-stack.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "kube-prometheus-stack.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }} +chart: {{ template "kube-prometheus-stack.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.operator.fullname" .) .Values.prometheusOperator.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (printf "%s-webhook" (include "kube-prometheus-stack.operator.fullname" .)) .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of prometheus service account to use */}} +{{- define "kube-prometheus-stack.prometheus.serviceAccountName" -}} +{{- if .Values.prometheus.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-prometheus") .Values.prometheus.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of alertmanager service account to use */}} +{{- define "kube-prometheus-stack.alertmanager.serviceAccountName" -}} +{{- if .Values.alertmanager.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-alertmanager") .Values.alertmanager.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.alertmanager.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of thanosRuler service account to use */}} +{{- define "kube-prometheus-stack.thanosRuler.serviceAccountName" -}} +{{- if .Values.thanosRuler.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.thanosRuler.name" .) .Values.thanosRuler.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.thanosRuler.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the grafana namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-grafana.namespace" -}} + {{- if .Values.grafana.namespaceOverride -}} + {{- .Values.grafana.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Allow kube-state-metrics job name to be overridden +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.name" -}} + {{- if index .Values "kube-state-metrics" "nameOverride" -}} + {{- index .Values "kube-state-metrics" "nameOverride" -}} + {{- else -}} + {{- print "kube-state-metrics" -}} + {{- end -}} +{{- end -}} + +{{/* +Use the kube-state-metrics namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.namespace" -}} + {{- if index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the prometheus-node-exporter namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-prometheus-node-exporter.namespace" -}} + {{- if index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Allow KubeVersion to be overridden. */}} +{{- define "kube-prometheus-stack.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{/* Get Ingress API Version */}} +{{- define "kube-prometheus-stack.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* Check Ingress stability */}} +{{- define "kube-prometheus-stack.ingress.isStable" -}} + {{- eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* Check Ingress supports pathType */}} +{{/* pathType was added to networking.k8s.io/v1beta1 in Kubernetes 1.18 */}} +{{- define "kube-prometheus-stack.ingress.supportsPathType" -}} + {{- or (eq (include "kube-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "kube-prometheus-stack.kubeVersion" .))) -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "kube-prometheus-stack.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} + {{- end -}} + +{{/* Get value based on current Kubernetes version */}} +{{- define "kube-prometheus-stack.kubeVersionDefaultValue" -}} + {{- $values := index . 0 -}} + {{- $kubeVersion := index . 1 -}} + {{- $old := index . 2 -}} + {{- $new := index . 3 -}} + {{- $default := index . 4 -}} + {{- if kindIs "invalid" $default -}} + {{- if semverCompare $kubeVersion (include "kube-prometheus-stack.kubeVersion" $values) -}} + {{- print $new -}} + {{- else -}} + {{- print $old -}} + {{- end -}} + {{- else -}} + {{- print $default }} + {{- end -}} +{{- end -}} + +{{/* Get value for kube-controller-manager depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeControllerManager.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.22-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Get value for kube-scheduler depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeScheduler.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.23-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +To help compatibility with other charts which use global.imagePullSecrets. +Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). +global: + imagePullSecrets: + - name: pullSecret1 + - name: pullSecret2 + +or + +global: + imagePullSecrets: + - pullSecret1 + - pullSecret2 +*/}} +{{- define "kube-prometheus-stack.imagePullSecrets" -}} +{{- range .Values.global.imagePullSecrets }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{- define "kube-prometheus-stack.operator.admission-webhook.dnsNames" }} +{{- $fullname := include "kube-prometheus-stack.operator.fullname" . }} +{{- $namespace := include "kube-prometheus-stack.namespace" . }} +{{- $fullname }} +{{ $fullname }}.{{ $namespace }}.svc +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +{{ $fullname }}-webhook +{{ $fullname }}-webhook.{{ $namespace }}.svc +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/alertmanager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/alertmanager.yaml new file mode 100644 index 0000000000..19044054ac --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/alertmanager.yaml @@ -0,0 +1,191 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.annotations }} + annotations: +{{ toYaml .Values.alertmanager.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.alertmanagerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.alertmanager.alertmanagerSpec.image.registry }} + {{- if and .Values.alertmanager.alertmanagerSpec.image.tag .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}" + {{- end }} + version: {{ .Values.alertmanager.alertmanagerSpec.image.tag }} + {{- if .Values.alertmanager.alertmanagerSpec.image.sha }} + sha: {{ .Values.alertmanager.alertmanagerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.alertmanager.alertmanagerSpec.replicas }} + listenLocal: {{ .Values.alertmanager.alertmanagerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.alertmanager.alertmanagerSpec.automountServiceAccountToken }} +{{- if .Values.alertmanager.alertmanagerSpec.externalUrl }} + externalUrl: "{{ tpl .Values.alertmanager.alertmanagerSpec.externalUrl . }}" +{{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.alertmanager.ingress.hosts 0) . }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ .Values.namespaceOverride }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.nodeSelector }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.alertmanager.alertmanagerSpec.paused }} + logFormat: {{ .Values.alertmanager.alertmanagerSpec.logFormat | quote }} + logLevel: {{ .Values.alertmanager.alertmanagerSpec.logLevel | quote }} + retention: {{ .Values.alertmanager.alertmanagerSpec.retention | quote }} +{{- if .Values.alertmanager.alertmanagerSpec.secrets }} + secrets: +{{ toYaml .Values.alertmanager.alertmanagerSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configSecret }} + configSecret: {{ .Values.alertmanager.alertmanagerSpec.configSecret }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configMaps }} + configMaps: +{{ toYaml .Values.alertmanager.alertmanagerSpec.configMaps | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector }} + alertmanagerConfigSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector | indent 4) . }} +{{ else }} + alertmanagerConfigSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector }} + alertmanagerConfigNamespaceSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector | indent 4) . }} +{{ else }} + alertmanagerConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.web }} + web: +{{ toYaml .Values.alertmanager.alertmanagerSpec.web | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration }} + alertmanagerConfiguration: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy }} + alertmanagerConfigMatcherStrategy: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.resources }} + resources: +{{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + routePrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.alertmanagerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.storage }} + storage: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.storage | indent 4) . }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.alertmanager.alertmanagerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.alertmanager.alertmanagerSpec.podAntiAffinity .Values.alertmanager.alertmanagerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.affinity }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.tolerations }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.containers }} + containers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.initContainers }} + initContainers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.priorityClassName }} + priorityClassName: {{.Values.alertmanager.alertmanagerSpec.priorityClassName }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.additionalPeers }} + additionalPeers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.additionalPeers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumes }} + volumes: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.alertmanager.alertmanagerSpec.portName }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} + clusterAdvertiseAddress: {{ .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} + clusterGossipInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} + clusterPeerTimeout: {{ .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} + clusterPushpullInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} + forceEnableClusterMode: {{ .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.minReadySeconds }} + minReadySeconds: {{ .Values.alertmanager.alertmanagerSpec.minReadySeconds }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/extrasecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/extrasecret.yaml new file mode 100644 index 0000000000..ecd8f47021 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.alertmanager.extraSecret.data -}} +{{- $secretName := printf "alertmanager-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.alertmanager.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.extraSecret.annotations }} + annotations: +{{ toYaml .Values.alertmanager.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.alertmanager.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/ingress.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/ingress.yaml new file mode 100644 index 0000000000..be9f5aa279 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/ingress.yaml @@ -0,0 +1,78 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} +{{- $pathType := .Values.alertmanager.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $backendServiceName := .Values.alertmanager.ingress.serviceName | default (printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager") }} +{{- $servicePort := .Values.alertmanager.ingress.servicePort | default .Values.alertmanager.service.port -}} +{{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} +{{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.alertmanager.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.alertmanager.ingress.labels }} +{{ toYaml .Values.alertmanager.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.alertmanager.ingress.ingressClassName }} + ingressClassName: {{ .Values.alertmanager.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.alertmanager.ingress.hosts }} + {{- range $host := .Values.alertmanager.ingress.hosts }} + - host: {{ tpl $host $ | quote }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.alertmanager.ingress.tls }} + tls: +{{ tpl (toYaml .Values.alertmanager.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/ingressperreplica.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/ingressperreplica.yaml new file mode 100644 index 0000000000..b2e00a4162 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled .Values.alertmanager.ingressPerReplica.enabled }} +{{- $pathType := .Values.alertmanager.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $servicePort := .Values.alertmanager.service.port -}} +{{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/podDisruptionBudget.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/podDisruptionBudget.yaml new file mode 100644 index 0000000000..b183403125 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.alertmanager.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.alertmanager.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/psp-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/psp-role.yaml new file mode 100644 index 0000000000..8810e93ded --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/psp-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/psp-rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/psp-rolebinding.yaml new file mode 100644 index 0000000000..794f4ad178 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/psp-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/psp.yaml new file mode 100644 index 0000000000..07b616b5cb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/psp.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/secret.yaml new file mode 100644 index 0000000000..d2fe84a7bf --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/secret.yaml @@ -0,0 +1,35 @@ +{{- if and (.Values.alertmanager.enabled) (not .Values.alertmanager.alertmanagerSpec.useExistingSecret) }} +{{/* This file is applied when the operation is helm install and the target secret does not exist. */}} +{{- $secretName := (printf "alertmanager-%s" (include "kube-prometheus-stack.alertmanager.crname" .)) }} +{{- if or (not (lookup "v1" "Secret" (include "kube-prometheus-stack.namespace" .) $secretName)) (eq .Values.alertmanager.secret.recreateIfExists true) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/resource-policy": keep +{{- if .Values.alertmanager.secret.annotations }} +{{ toYaml .Values.alertmanager.secret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if .Values.alertmanager.tplConfig }} +{{- if .Values.alertmanager.stringConfig }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.stringConfig) . | b64enc | quote }} +{{- else if eq (typeOf .Values.alertmanager.config) "string" }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.config) . | b64enc | quote }} +{{- else }} + alertmanager.yaml: {{ tpl (toYaml .Values.alertmanager.config) . | b64enc | quote }} +{{- end }} +{{- else }} + alertmanager.yaml: {{ toYaml .Values.alertmanager.config | b64enc | quote }} +{{- end }} +{{- range $key, $val := .Values.alertmanager.templateFiles }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/service.yaml new file mode 100644 index 0000000000..373de328a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/service.yaml @@ -0,0 +1,68 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.alertmanager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + self-monitor: {{ .Values.alertmanager.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.service.labels }} +{{ toYaml .Values.alertmanager.service.labels | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.annotations }} + annotations: +{{ toYaml .Values.alertmanager.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.service.clusterIP }} + clusterIP: {{ .Values.alertmanager.service.clusterIP }} +{{- end }} +{{- if .Values.alertmanager.service.externalIPs }} + externalIPs: +{{ toYaml .Values.alertmanager.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.alertmanager.service.loadBalancerIP }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.alertmanager.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.alertmanager.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq .Values.alertmanager.service.type "NodePort" }} + nodePort: {{ .Values.alertmanager.service.nodePort }} + {{- end }} + port: {{ .Values.alertmanager.service.port }} + targetPort: {{ .Values.alertmanager.service.targetPort }} + protocol: TCP + - name: reloader-web + {{- if semverCompare ">=1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: 8080 + targetPort: reloader-web +{{- if .Values.alertmanager.service.additionalPorts }} +{{ toYaml .Values.alertmanager.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- if .Values.alertmanager.service.sessionAffinity }} + sessionAffinity: {{ .Values.alertmanager.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.alertmanager.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.alertmanager.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.alertmanager.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/serviceaccount.yaml new file mode 100644 index 0000000000..745ced8bde --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.alertmanager.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.alertmanager.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2}} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/servicemonitor.yaml new file mode 100644 index 0000000000..6233690019 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/servicemonitor.yaml @@ -0,0 +1,84 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.alertmanager.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.alertmanager.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + enableHttp2: {{ .Values.alertmanager.serviceMonitor.enableHttp2 }} + {{- if .Values.alertmanager.serviceMonitor.interval }} + interval: {{ .Values.alertmanager.serviceMonitor.interval }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.alertmanager.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.scheme }} + scheme: {{ .Values.alertmanager.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.alertmanager.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.alertmanager.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.alertmanager.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.alertmanager.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.alertmanager.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.alertmanager.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.alertmanager.serviceMonitor.interval .interval }} + interval: {{ default $.Values.alertmanager.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.alertmanager.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.alertmanager.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/serviceperreplica.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/serviceperreplica.yaml new file mode 100644 index 0000000000..75a13bdf97 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/alertmanager/serviceperreplica.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $serviceValues := .Values.alertmanager.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" $ }} + statefulset.kubernetes.io/pod-name: alertmanager-{{ include "kube-prometheus-stack.alertmanager.crname" $ }}-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/core-dns/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/core-dns/service.yaml new file mode 100644 index 0000000000..b8618f7558 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/core-dns/service.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + jobLabel: coredns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.coreDns.serviceMonitor.port }} + port: {{ .Values.coreDns.service.port }} + protocol: TCP + targetPort: {{ .Values.coreDns.service.targetPort }} + selector: + {{- if .Values.coreDns.service.selector }} +{{ toYaml .Values.coreDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/core-dns/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/core-dns/servicemonitor.yaml new file mode 100644 index 0000000000..dc15a06937 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/core-dns/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + {{- with .Values.coreDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.coreDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.coreDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.coreDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.coreDns.serviceMonitor.port }} + {{- if .Values.coreDns.serviceMonitor.interval}} + interval: {{ .Values.coreDns.serviceMonitor.interval }} + {{- end }} + {{- if .Values.coreDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.coreDns.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.coreDns.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.coreDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.coreDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml new file mode 100644 index 0000000000..66e777632e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.kubeApiServer.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-apiserver + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: default + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-apiserver + {{- with .Values.kubeApiServer.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubeApiServer.serviceMonitor | nindent 2 }} + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeApiServer.serviceMonitor.interval }} + interval: {{ .Values.kubeApiServer.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubeApiServer.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeApiServer.serviceMonitor.proxyUrl }} + {{- end }} + port: https + scheme: https + metricRelabelings: + {{- if .Values.kubeApiServer.serviceMonitor.metricRelabelings }} +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeApiServer.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.relabelings | indent 6) . }} +{{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + serverName: {{ .Values.kubeApiServer.tlsConfig.serverName }} + insecureSkipVerify: {{ .Values.kubeApiServer.tlsConfig.insecureSkipVerify }} + jobLabel: {{ .Values.kubeApiServer.serviceMonitor.jobLabel }} + namespaceSelector: + matchNames: + - default + selector: +{{ toYaml .Values.kubeApiServer.serviceMonitor.selector | indent 4 }} +{{- end}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml new file mode 100644 index 0000000000..6a6afa6412 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + k8s-app: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeControllerManager.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-controller-manager/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-controller-manager/service.yaml new file mode 100644 index 0000000000..43b1a976d5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-controller-manager/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + jobLabel: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.targetPort) }} +{{- if .Values.kubeControllerManager.endpoints }}{{- else }} + selector: + {{- if .Values.kubeControllerManager.service.selector }} +{{ toYaml .Values.kubeControllerManager.service.selector | indent 4 }} + {{- else}} + component: kube-controller-manager + {{- end}} +{{- end }} + type: ClusterIP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml new file mode 100644 index 0000000000..7ed3baa65f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + {{- with .Values.kubeControllerManager.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeControllerManager.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeControllerManager.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeControllerManager.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- if .Values.kubeControllerManager.serviceMonitor.interval }} + interval: {{ .Values.kubeControllerManager.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeControllerManager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeControllerManager.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . false true .Values.kubeControllerManager.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . nil true .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeControllerManager.serviceMonitor.serverName }} + serverName: {{ .Values.kubeControllerManager.serviceMonitor.serverName }} + {{- end }} + {{- end }} + metricRelabelings: + {{- if.Values.kubeControllerManager.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeControllerManager.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-dns/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-dns/service.yaml new file mode 100644 index 0000000000..81b2c9930c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-dns/service.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + jobLabel: kube-dns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics-dnsmasq + port: {{ .Values.kubeDns.service.dnsmasq.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.dnsmasq.targetPort }} + - name: http-metrics-skydns + port: {{ .Values.kubeDns.service.skydns.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.skydns.targetPort }} + selector: + {{- if .Values.kubeDns.service.selector }} +{{ toYaml .Values.kubeDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml new file mode 100644 index 0000000000..9fa41b575f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + {{- with .Values.kubeDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics-dnsmasq + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeDns.serviceMonitor.proxyUrl}} + {{- end }} + metricRelabelings: + {{- if .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeDns.serviceMonitor.dnsmasqRelabelings }} + relabelings: +{{ toYaml .Values.kubeDns.serviceMonitor.dnsmasqRelabelings | indent 4 }} +{{- end }} + - port: http-metrics-skydns + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubeDns.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-etcd/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-etcd/endpoints.yaml new file mode 100644 index 0000000000..e366447577 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-etcd/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + k8s-app: etcd-server +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeEtcd.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-etcd/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-etcd/service.yaml new file mode 100644 index 0000000000..d07d4f35e3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-etcd/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + jobLabel: kube-etcd +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeEtcd.service.targetPort }} +{{- if .Values.kubeEtcd.endpoints }}{{- else }} + selector: + {{- if .Values.kubeEtcd.service.selector }} +{{ toYaml .Values.kubeEtcd.service.selector | indent 4 }} + {{- else}} + component: etcd + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml new file mode 100644 index 0000000000..26fdbdbed3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + {{- with .Values.kubeEtcd.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeEtcd.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeEtcd.serviceMonitor | nindent 4 }} + selector: + {{- if .Values.kubeEtcd.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeEtcd.serviceMonitor.port }} + {{- if .Values.kubeEtcd.serviceMonitor.interval }} + interval: {{ .Values.kubeEtcd.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeEtcd.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeEtcd.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq .Values.kubeEtcd.serviceMonitor.scheme "https" }} + scheme: https + tlsConfig: + {{- if .Values.kubeEtcd.serviceMonitor.serverName }} + serverName: {{ .Values.kubeEtcd.serviceMonitor.serverName }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.caFile }} + caFile: {{ .Values.kubeEtcd.serviceMonitor.caFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.certFile }} + certFile: {{ .Values.kubeEtcd.serviceMonitor.certFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.keyFile }} + keyFile: {{ .Values.kubeEtcd.serviceMonitor.keyFile }} + {{- end}} + insecureSkipVerify: {{ .Values.kubeEtcd.serviceMonitor.insecureSkipVerify }} + {{- end }} + metricRelabelings: + {{- if .Values.kubeEtcd.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeEtcd.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-proxy/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-proxy/endpoints.yaml new file mode 100644 index 0000000000..8613e62425 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-proxy/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + k8s-app: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeProxy.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-proxy/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-proxy/service.yaml new file mode 100644 index 0000000000..8ccb2210d7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-proxy/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + jobLabel: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeProxy.service.targetPort }} +{{- if .Values.kubeProxy.endpoints }}{{- else }} + selector: + {{- if .Values.kubeProxy.service.selector }} +{{ toYaml .Values.kubeProxy.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-proxy + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml new file mode 100644 index 0000000000..24b0ab2001 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + {{- with .Values.kubeProxy.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeProxy.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeProxy.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeProxy.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeProxy.serviceMonitor.port }} + {{- if .Values.kubeProxy.serviceMonitor.interval }} + interval: {{ .Values.kubeProxy.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeProxy.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeProxy.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.kubeProxy.serviceMonitor.https }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- end}} + metricRelabelings: + {{- if .Values.kubeProxy.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeProxy.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeProxy.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml new file mode 100644 index 0000000000..6236b42f10 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + k8s-app: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeScheduler.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-scheduler/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-scheduler/service.yaml new file mode 100644 index 0000000000..90b3a800a4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-scheduler/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + jobLabel: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.targetPort) }} +{{- if .Values.kubeScheduler.endpoints }}{{- else }} + selector: + {{- if .Values.kubeScheduler.service.selector }} +{{ toYaml .Values.kubeScheduler.service.selector | indent 4 }} + {{- else}} + component: kube-scheduler + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml new file mode 100644 index 0000000000..b17c4f1d47 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + {{- with .Values.kubeScheduler.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeScheduler.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeScheduler.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeScheduler.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- if .Values.kubeScheduler.serviceMonitor.interval }} + interval: {{ .Values.kubeScheduler.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeScheduler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeScheduler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . false true .Values.kubeScheduler.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . nil true .Values.kubeScheduler.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeScheduler.serviceMonitor.serverName }} + serverName: {{ .Values.kubeScheduler.serviceMonitor.serverName }} + {{- end}} + {{- end}} + metricRelabelings: + {{- if .Values.kubeScheduler.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeScheduler.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-state-metrics/validate.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-state-metrics/validate.yaml new file mode 100644 index 0000000000..9211b3d771 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kube-state-metrics/validate.yaml @@ -0,0 +1,7 @@ +{{- if .Values.kubeStateMetrics.enabled }} +{{- if not (kindIs "invalid" .Values.kubeStateMetrics.serviceMonitor) }} +{{- if .Values.kubeStateMetrics.serviceMonitor.namespaceOverride }} +{{- fail "kubeStateMetrics.serviceMonitor.namespaceOverride was removed. Please use kube-state-metrics.namespaceOverride instead." }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kubelet/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kubelet/servicemonitor.yaml new file mode 100644 index 0000000000..f570fbfdbc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/kubelet/servicemonitor.yaml @@ -0,0 +1,246 @@ +{{- if (and (not .Values.kubelet.enabled) .Values.hardenedKubelet.enabled) }} +{{ required "Cannot set .Values.hardenedKubelet.enabled=true when .Values.kubelet.enabled=false" "" }} +{{- end }} +{{- if (and .Values.kubelet.enabled .Values.kubernetesServiceMonitors.enabled (not .Values.hardenedKubelet.enabled) (not .Values.k3sServer.enabled)) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kubelet + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: {{ .Values.kubelet.namespace }} + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kubelet + {{- with .Values.kubelet.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubelet.serviceMonitor | nindent 2 }} + {{- with .Values.kubelet.serviceMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + {{- if .Values.kubelet.serviceMonitor.https }} + - port: https-metrics + scheme: https + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + metricRelabelings: + {{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: https-metrics + scheme: https + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: https-metrics + scheme: https + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: https-metrics + scheme: https + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} + {{- else }} + - port: http-metrics + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: http-metrics + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: http-metrics + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: http-metrics + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- end }} + {{- end }} + jobLabel: k8s-app + namespaceSelector: + matchNames: + - {{ .Values.kubelet.namespace }} + selector: + matchLabels: + app.kubernetes.io/name: kubelet + k8s-app: kubelet +{{- end}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/node-exporter/validate.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/node-exporter/validate.yaml new file mode 100644 index 0000000000..bdc73d6165 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/exporters/node-exporter/validate.yaml @@ -0,0 +1,3 @@ +{{- if (and (not .Values.nodeExporter.enabled) .Values.hardenedNodeExporter.enabled) }} +{{ required "Cannot set .Values.hardenedNodeExporter.enabled=true when .Values.nodeExporter.enabled=false" "" }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/extra-objects.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/extra-objects.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/extra-objects.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/configmap-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/configmap-dashboards.yaml new file mode 100644 index 0000000000..e719009ffe --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/configmap-dashboards.yaml @@ -0,0 +1,24 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled) .Values.grafana.forceDeployDashboards }} +{{- $files := .Files.Glob "dashboards-1.14/*.json" }} +{{- if $files }} +apiVersion: v1 +kind: ConfigMapList +items: +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 6 }} + data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/configmaps-datasources.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/configmaps-datasources.yaml new file mode 100644 index 0000000000..718020d4f6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/configmaps-datasources.yaml @@ -0,0 +1,81 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled) .Values.grafana.forceDeployDatasources }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-grafana-datasource + namespace: {{ default .Values.grafana.sidecar.datasources.searchNamespace (include "kube-prometheus-stack.namespace" .) }} +{{- if .Values.grafana.sidecar.datasources.annotations }} + annotations: + {{- toYaml .Values.grafana.sidecar.datasources.annotations | nindent 4 }} +{{- end }} + labels: + {{ $.Values.grafana.sidecar.datasources.label }}: {{ $.Values.grafana.sidecar.datasources.labelValue | quote }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + datasource.yaml: |- + apiVersion: 1 +{{- if .Values.grafana.deleteDatasources }} + deleteDatasources: +{{ tpl (toYaml .Values.grafana.deleteDatasources | indent 6) . }} +{{- end }} + datasources: +{{- $scrapeInterval := .Values.grafana.sidecar.datasources.defaultDatasourceScrapeInterval | default .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} +{{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} + - name: Prometheus + type: prometheus + uid: {{ .Values.grafana.sidecar.datasources.uid }} + {{- if .Values.grafana.sidecar.datasources.url }} + url: {{ .Values.grafana.sidecar.datasources.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + {{- end }} + access: proxy + isDefault: {{ .Values.grafana.sidecar.datasources.isDefaultDatasource }} + jsonData: + httpMethod: {{ .Values.grafana.sidecar.datasources.httpMethod }} + timeInterval: {{ $scrapeInterval }} + {{- if .Values.grafana.sidecar.datasources.timeout }} + timeout: {{ .Values.grafana.sidecar.datasources.timeout }} + {{- end }} +{{- if .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.createPrometheusReplicasDatasources }} +{{- range until (int .Values.prometheus.prometheusSpec.replicas) }} + - name: Prometheus-{{ . }} + type: prometheus + uid: {{ $.Values.grafana.sidecar.datasources.uid }}-replica-{{ . }} + url: http://prometheus-{{ template "kube-prometheus-stack.prometheus.crname" $ }}-{{ . }}.prometheus-operated:9090/{{ trimPrefix "/" $.Values.prometheus.prometheusSpec.routePrefix }} + access: proxy + isDefault: false + jsonData: + timeInterval: {{ $scrapeInterval }} +{{- if $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.alertmanager.enabled }} + - name: Alertmanager + type: alertmanager + uid: {{ .Values.grafana.sidecar.datasources.alertmanager.uid }} + {{- if .Values.grafana.sidecar.datasources.alertmanager.url }} + url: {{ .Values.grafana.sidecar.datasources.alertmanager.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }}/{{ trimPrefix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }} + {{- end }} + access: proxy + jsonData: + handleGrafanaManagedAlerts: {{ .Values.grafana.sidecar.datasources.alertmanager.handleGrafanaManagedAlerts }} + implementation: {{ .Values.grafana.sidecar.datasources.alertmanager.implementation }} +{{- end }} +{{- end }} +{{- if .Values.grafana.additionalDataSources }} +{{ tpl (toYaml .Values.grafana.additionalDataSources | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml new file mode 100644 index 0000000000..dfc26d7ecd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml @@ -0,0 +1,616 @@ +{{- /* +Generated from 'alertmanager-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "alertmanager-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + alertmanager-overview.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "current set of alerts stored in the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid alerts received by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Received", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Invalid", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts receive rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alerts", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Failed", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notifications Send Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "latency of notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} 99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Median", + "refId": "B" + }, + { + "expr": "sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notification Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Notifications", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "alertmanager-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "service", + "multi": false, + "name": "service", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, service)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "integration", + "options": [ + + ], + "query": "label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Alertmanager / Overview", + "uid": "alertmanager-overview", + "version": 0 + } +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml new file mode 100644 index 0000000000..bd1048b567 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml @@ -0,0 +1,1772 @@ +{{- /* +Generated from 'apiserver' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeApiServer.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "apiserver" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + apiserver.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "datasource": null, + "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "mode": "markdown", + "span": 12, + "title": "Notice", + "type": "text" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Availability (30d) > 99.000%", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "description": "How much error budget is left looking at our 0.990% availability guarantees?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "errorbudget", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ErrorBudget (30d) > 99.000%", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Read Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many read requests (LIST,GET) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Write Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / API server", + "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml new file mode 100644 index 0000000000..f4be0bbd45 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -0,0 +1,1882 @@ +{{- /* +Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "cluster-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + cluster-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "namespace", + "value": "namespace" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth History", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "What is TCP Retransmit?", + "url": "https://accedian.com/enterprises/blog/network-packet-loss-retransmissions-and-duplicate-acknowledgements/" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP Retransmits out of all sent segments", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Why monitor SYN retransmits?", + "url": "https://github.com/prometheus/node_exporter/issues/1023#issuecomment-408128365" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP SYN Retransmits out of all retransmits", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Cluster", + "uid": "ff635a025bcfea7bc3dd4f508990a3e9", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml new file mode 100644 index 0000000000..8d420d7a4f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -0,0 +1,1196 @@ +{{- /* +Generated from 'controller-manager' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "controller-manager" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + controller-manager.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Controller Manager", + "uid": "72e0e05bef5099e5f049b05fdc429ed4", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml new file mode 100644 index 0000000000..0eeedc6299 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml @@ -0,0 +1,1229 @@ +{{- /* +Generated from 'etcd' from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "etcd" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + etcd.json: |- + { + "annotations": { + "list": [] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 42, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"$cluster\"}[$__rate_interval])))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer round trip time", + "metric": "etcd_network_peer_round_trip_time_seconds_bucket", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Peer round trip time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:925", + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:926", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + "etcd-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": {{ if (or .Values.grafana.sidecar.dashboards.multicluster.global.enabled .Values.grafana.sidecar.dashboards.multicluster.etcd.enabled) }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 2, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "etcd", + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", + "version": 215 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml new file mode 100644 index 0000000000..d2609140cf --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml @@ -0,0 +1,635 @@ +{{- /* +Generated from 'grafana-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "grafana-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + grafana-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + + ], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3085, + "iteration": 1631554945276, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Firing Alerts", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dashboards", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "displayMode": "auto" + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "showHeader": true + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_build_info{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Build Info", + "transformations": [ + { + "id": "labelsToFields", + "options": { + + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "branch": true, + "container": true, + "goversion": true, + "namespace": true, + "pod": true, + "revision": true + }, + "indexByName": { + "Time": 7, + "Value": 11, + "branch": 4, + "container": 8, + "edition": 2, + "goversion": 6, + "instance": 1, + "job": 0, + "namespace": 9, + "pod": 10, + "revision": 5, + "version": 3 + }, + "renameByName": { + + } + } + } + ], + "type": "table" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ", + "interval": "", + "legendFormat": "{{`{{`}}status_code{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "RPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:157", + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:210", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:211", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 30, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": [ + "default/grafana" + ], + "value": [ + "default/grafana" + ] + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, job)", + "refId": "Billing Admin-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "instance", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, instance)", + "refId": "Billing Admin-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Grafana Overview", + "uid": "6be0s85Mk", + "version": 2 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml new file mode 100644 index 0000000000..7ecca76f23 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml @@ -0,0 +1,1534 @@ +{{- /* +Generated from 'k8s-coredns' from ../files/dashboards/k8s-coredns.json +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.coreDns.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-coredns" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-coredns.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for the CoreDNS DNS server with updated metrics for version 1.7.0+. Based on the CoreDNS dashboard by buhay.", + "editable": true, + "gnetId": 12539, + "graphTooltip": 0, + "iteration": 1603798405693, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "CoreDNS.io", + "type": "link", + "url": "https://coredns.io" + } + ], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (total)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + }, + { + "alias": "other", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type) or \nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by qtype)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{zone}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by zone)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_do_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_do_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DO", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (DO bit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 2 + }, + { + "alias": "tcp:99 ", + "yaxis": 2 + }, + { + "alias": "tcp:50", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 1 + }, + { + "alias": "tcp:99 ", + "yaxis": 1 + }, + { + "alias": "tcp:50", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size,tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode) or\nsum(rate(coredns_dns_responses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{rcode}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (by rcode)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le, job))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (duration)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + }, + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{job=\"coredns\",instance=~\"$instance\"}) by (type) or\nsum(coredns_cache_entries{job=\"coredns\",instance=~\"$instance\"}) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (size)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "misses", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "hits:{{"{{type}}"}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_cache_misses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "misses", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (hitrate)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "dns", + "coredns" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(up{job=\"coredns\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=\"coredns\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "CoreDNS", + "uid": "vkQ0UHxik", + "version": 2 + } +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml new file mode 100644 index 0000000000..93ee57db93 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -0,0 +1,3088 @@ +{{- /* +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Namespace", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 20, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 21, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 22, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml new file mode 100644 index 0000000000..9c295831a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-multicluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-multicluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-multicluster.json: |- + {{`{"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster:node_cpu:ratio_rate5m","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Cluster","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Multi-Cluster","uid":"b59e6c9f2fcbe2e16d77fc492374cc4f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml new file mode 100644 index 0000000000..1c32c9c02e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -0,0 +1,2797 @@ +{{- /* +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml new file mode 100644 index 0000000000..e60a42d747 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -0,0 +1,1026 @@ +{{- /* +Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-node" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-node.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml new file mode 100644 index 0000000000..80fab51c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -0,0 +1,2469 @@ +{{- /* +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.25, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Throttling", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Throttling", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (WSS)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage (WSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Pod - Read & Writes)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Containers)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml new file mode 100644 index 0000000000..d77170afd8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-cluster' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-cluster.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - avg(rate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode=\"idle\"}[1m]))","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:windows_node_memory_MemFreeCached_bytes:sum{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Private Working Set)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Namespace","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Cluster(Windows)","uid":"4d08557fd9391b100730f2494bccac68","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml new file mode 100644 index 0000000000..13a1fc3abd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-namespace' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-namespace.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Namespace(Windows)","uid":"490b402361724ab1d4c45666c1fa9b6f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml new file mode 100644 index 0000000000..6686e54053 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-pod' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-pod.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":6,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum by (container) (rate(windows_container_network_received_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Received : {{ container }}","refId":"A"},{"expr":"sort_desc(sum by (container) (rate(windows_container_network_transmitted_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Transmitted : {{ container }}","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network I/O","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Pod","multi":false,"name":"pod","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\",namespace=\"$namespace\"}, pod)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Pod(Windows)","uid":"40597a704a610e936dc6ed374a7ce023","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml new file mode 100644 index 0000000000..e2a63ae208 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -0,0 +1,2024 @@ +{{- /* +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Pod", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}, workload)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml new file mode 100644 index 0000000000..95d758ea2d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -0,0 +1,2189 @@ +{{- /* +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Workload", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..d9ce9d738c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-cluster-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-cluster-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\"} * node:windows_node_num_cpu:sum{cluster=\"$cluster\"} / scalar(sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"}))","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:ratio{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O Pages)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(node:windows_node:sum{cluster=\"$cluster\"})","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (instance)(node:windows_node_filesystem_usage:{cluster=\"$cluster\"})\n","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Capacity","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Cluster(Windows)","uid":"53a43377ec9aaf2ff64dfc7a1f539334","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml new file mode 100644 index 0000000000..a7608496a3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-node-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-node-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (core) (irate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode!=\"idle\", instance=\"$instance\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{core}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage Per Core","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Memory","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation %","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"max(\n windows_os_visible_memory_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n - windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"memory used","refId":"A"},{"expr":"max(node:windows_node_memory_totalCached_bytes:sum{cluster=\"$cluster\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory cached","refId":"B"},{"expr":"max(windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory free","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Swap IO","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O) Pages","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"read","yaxis":1},{"alias":"io time","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"max(rate(windows_logical_disk_read_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"read","refId":"A"},{"expr":"max(rate(windows_logical_disk_write_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"written","refId":"B"},{"expr":"max(rate(windows_logical_disk_read_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]) + rate(windows_logical_disk_write_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ms","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Saturation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Net","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}\n","format":"time_series","legendFormat":"{{volume}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(windows_system_system_up_time{cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Node(Windows)","uid":"96e7484b0bb53b74fbc2bcb7723cd40b","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml new file mode 100644 index 0000000000..74a5303f8f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml @@ -0,0 +1,2256 @@ +{{- /* +Generated from 'kubelet' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubelet.enabled" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "kubelet" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + kubelet.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_node_name{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Running Kubelets", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Pods", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Containers", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 5, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Actual Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Desired Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 7, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Config Error Count", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 77 + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 77 + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 77 + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Kubelet", + "uid": "3138fa155d5915769fbded898ac09fd9", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml new file mode 100644 index 0000000000..f5c72844fb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -0,0 +1,1464 @@ +{{- /* +Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-pod.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "pod", + "value": "pod" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Pods)", + "uid": "8b7a8b326d7a6f1f04244066368c67af", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml new file mode 100644 index 0000000000..801b09c265 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -0,0 +1,1736 @@ +{{- /* +Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-workload.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "workload", + "value": "workload" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=$__cell", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Workload)", + "uid": "bbb2a765a623ae38130206c7d94a160f", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..9869a3d3e0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -0,0 +1,1063 @@ +{{- /* +Generated from 'node-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-cluster-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "((\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} instance {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"})))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Cluster", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml new file mode 100644 index 0000000000..75e69afa22 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -0,0 +1,1089 @@ +{{- /* +Generated from 'node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major page Faults", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_exporter_build_info{job=\"node-exporter\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Node", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml new file mode 100644 index 0000000000..fe11875324 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml @@ -0,0 +1,1073 @@ +{{- /* +Generated from 'nodes-darwin' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.darwin.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes-darwin" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes-darwin.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Physical Memory", + "refId": "A" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory Used", + "refId": "B" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "App Memory", + "refId": "C" + }, + { + "expr": "node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Wired Memory", + "refId": "D" + }, + { + "expr": "node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compressed", + "refId": "E" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "(\n (\n avg(node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"}) -\n avg(node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"})\n ) /\n avg(node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"})\n)\n*\n100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / MacOS", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml new file mode 100644 index 0000000000..0da40a7b99 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml @@ -0,0 +1,1066 @@ +{{- /* +Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.linux.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"})\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname!=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / Nodes", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml new file mode 100644 index 0000000000..4d1e33208b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -0,0 +1,587 @@ +{{- /* +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml new file mode 100644 index 0000000000..9a7e7d0603 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml @@ -0,0 +1,1228 @@ +{{- /* +Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "pod-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + pod-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 8, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Pod", + "uid": "7a18067ce943a40ae25454675c19ff5c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml new file mode 100644 index 0000000000..5c11900e69 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -0,0 +1,1674 @@ +{{- /* +Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus-remote-write" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus-remote-write.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} != 0)\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n, 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Min Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Desired Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shards", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shard Details", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}consumer{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Remote Write Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Segments", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Retried Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Enqueue Retries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Misc. Rates", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": true, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "url", + "options": [ + + ], + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Remote Write", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml new file mode 100644 index 0000000000..27f7c44e2c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml @@ -0,0 +1,1235 @@ +{{- /* +Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "s" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}scrape_job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}interval{{`}}`}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded body size limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}slice{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=\"prometheus-k8s\",namespace=\"monitoring\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Overview", + "uid": "", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml new file mode 100644 index 0000000000..410812451e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml @@ -0,0 +1,1276 @@ +{{- /* +Generated from 'proxy' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "proxy" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + proxy.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rules Sync Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rule Sync Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Proxy", + "uid": "632e265de029684c40b21cb76bca4f94", + "version": 0 + } +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml new file mode 100644 index 0000000000..ee0cf08b2f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml @@ -0,0 +1,1118 @@ +{{- /* +Generated from 'scheduler' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "scheduler" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + scheduler.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml new file mode 100644 index 0000000000..5aafccdebe --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml @@ -0,0 +1,1438 @@ +{{- /* +Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "workload-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + workload-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 14, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Workload", + "uid": "728bf77cc1166d2f3133bf25846876cc", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/namespaces.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/namespaces.yaml new file mode 100644 index 0000000000..39ed210ed4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/grafana/namespaces.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + name: {{ .Values.grafana.defaultDashboards.namespace }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + annotations: +{{- if not .Values.grafana.defaultDashboards.cleanupOnUninstall }} + helm.sh/resource-policy: "keep" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl new file mode 100644 index 0000000000..6ae9dc72e6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl @@ -0,0 +1,7 @@ +{{/* Generate basic labels for prometheus-operator */}} +{{- define "kube-prometheus-stack.prometheus-operator.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app: {{ template "kube-prometheus-stack.name" . }}-operator +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl new file mode 100644 index 0000000000..f419caf54b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl @@ -0,0 +1,6 @@ +{{/* Generate basic labels for prometheus-operator-webhook */}} +{{- define "kube-prometheus-stack.prometheus-operator-webhook.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator-webhook +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml new file mode 100644 index 0000000000..054eac4a77 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml @@ -0,0 +1,143 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.annotations | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.prometheusOperator.admissionWebhooks.deployment.replicas }} + revisionHistoryLimit: {{ .Values.prometheusOperator.admissionWebhooks.deployment.revisionHistoryLimit }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podLabels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: prometheus-operator-admission-webhook + {{- $operatorRegistry := .Values.global.imageRegistry | default .Values.prometheusOperator.admissionWebhooks.deployment.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + - --log-format={{ .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + - --log-level={{ .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - "--web.enable-tls=true" + - "--web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }}" + - "--web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }}" + - "--web.listen-address=:{{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }}" + - "--web.tls-min-version={{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.tlsMinVersion }}" + ports: + - containerPort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.failureThreshold }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.containerSecurityContext | indent 12 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + volumeMounts: + - name: tls-secret + mountPath: /cert + readOnly: true + volumes: + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }}-webhook + automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml new file mode 100644 index 0000000000..52dd78f624 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget -}} +apiVersion: policy/v1{{ ternary "" "beta1" ($.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml new file mode 100644 index 0000000000..b06c129123 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.admissionWebhooks.deployment.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml new file mode 100644 index 0000000000..55511da36b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ template "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | indent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml new file mode 100644 index 0000000000..f7543b0f1a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml new file mode 100644 index 0000000000..4e3b0d9225 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml new file mode 100644 index 0000000000..b81257c168 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") (or .Values.global.cattle.psp.enabled .Values.global.rbac.pspEnabled) }} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} + - apiGroups: ['policy'] +{{- else }} + - apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml new file mode 100644 index 0000000000..4cf1335b22 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml new file mode 100644 index 0000000000..baed83db48 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -0,0 +1,73 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: create + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | replace "\n" "," }} + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + {{- with .Values.prometheusOperator.admissionWebhooks.createSecretJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml new file mode 100644 index 0000000000..5639cc9e80 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -0,0 +1,74 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: patch + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - patch + - --webhook-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --patch-failure-policy={{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- with .Values.prometheusOperator.admissionWebhooks.patchWebhookJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml new file mode 100644 index 0000000000..864deb52a0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml new file mode 100644 index 0000000000..076c467004 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml new file mode 100644 index 0000000000..0113b6a5d8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,47 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- if .Values.global.rbac.pspAnnotations }} +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml new file mode 100644 index 0000000000..f15abf4395 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml new file mode 100644 index 0000000000..30bde920b6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml new file mode 100644 index 0000000000..02594547d1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..da01f3b57e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/mutate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..4827871cca --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/validate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/certmanager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/certmanager.yaml new file mode 100644 index 0000000000..cb27e49f48 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/certmanager.yaml @@ -0,0 +1,55 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled -}} +{{- if not .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef -}} +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }} + issuerRef: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.kube-prometheus-stack" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + ca: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert +{{- end }} +--- +# generate a server certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }} + issuerRef: + {{- if .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef }} + {{- toYaml .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef | nindent 4 }} + {{- else }} + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + {{- end }} + dnsNames: + {{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | splitList "\n" | toYaml | nindent 4 }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..07e2e99967 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml @@ -0,0 +1,40 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort | quote }} + {{- else }} + - port: "8080" + {{- end }} + protocol: "TCP" + {{- if not .Values.prometheusOperator.tls.enabled }} + rules: + http: + - method: "GET" + path: "/metrics" + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/clusterrole.yaml new file mode 100644 index 0000000000..fd11b69eed --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/clusterrole.yaml @@ -0,0 +1,109 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - alertmanagers/finalizers + - alertmanagers/status + - alertmanagerconfigs + - prometheuses + - prometheuses/finalizers + - prometheuses/status + - prometheusagents + - prometheusagents/finalizers + - prometheusagents/status + - thanosrulers + - thanosrulers/finalizers + - thanosrulers/status + - scrapeconfigs + - servicemonitors + - podmonitors + - probes + - prometheusrules + verbs: + - '*' +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - '*' +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete +- apiGroups: + - "" + resources: + - services + - services/finalizers + - endpoints + verbs: + - get + - create + - update + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - patch + - create +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get +{{- if .Capabilities.APIVersions.Has "discovery.k8s.io/v1/EndpointSlice" }} +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml new file mode 100644 index 0000000000..ad9e3ef6c5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/deployment.yaml new file mode 100644 index 0000000000..8a01b2912a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/deployment.yaml @@ -0,0 +1,204 @@ +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- $defaultKubeletSvcName := printf "%s-kubelet" (include "kube-prometheus-stack.fullname" .) }} +{{- if .Values.prometheusOperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.labels }} +{{ toYaml .Values.prometheusOperator.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.annotations | indent 4 }} +{{- end }} +spec: + replicas: 1 + revisionHistoryLimit: {{ .Values.prometheusOperator.revisionHistoryLimit }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- with .Values.prometheusOperator.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.podLabels }} +{{ toYaml .Values.prometheusOperator.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: {{ template "kube-prometheus-stack.name" . }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- $configReloaderRegistry := $base_registry | default .Values.prometheusOperator.prometheusConfigReloader.image.registry -}} + {{- $operatorRegistry := $base_registry | default .Values.prometheusOperator.image.registry -}} + {{- $thanosRegistry := $base_registry | default .Values.prometheusOperator.thanosImage.registry -}} + {{- if .Values.prometheusOperator.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.kubeletService.enabled }} + - --kubelet-service={{ .Values.prometheusOperator.kubeletService.namespace }}/{{ default $defaultKubeletSvcName .Values.prometheusOperator.kubeletService.name }} + {{- end }} + {{- if .Values.prometheusOperator.logFormat }} + - --log-format={{ .Values.prometheusOperator.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.logLevel }} + - --log-level={{ .Values.prometheusOperator.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - --deny-namespaces={{ tpl (.Values.prometheusOperator.denyNamespaces | join ",") $ }} + {{- end }} + {{- with $.Values.prometheusOperator.namespaces }} + {{- $namespaces := list }} + {{- if .releaseNamespace }} + {{- $namespaces = append $namespaces $namespace }} + {{- end }} + {{- if .additional }} + {{- range $ns := .additional }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + - --localhost=127.0.0.1 + {{- if .Values.prometheusOperator.prometheusDefaultBaseImage }} + - --prometheus-default-base-image={{ $base_registry | default .Values.prometheusOperator.prometheusDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.prometheusDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + - --alertmanager-default-base-image={{ $base_registry | default .Values.prometheusOperator.alertmanagerDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + {{- else }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }} + {{- end }} + - --config-reloader-cpu-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).cpu) | default 0 }} + - --config-reloader-cpu-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).cpu) | default 0 }} + - --config-reloader-memory-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).memory) | default 0 }} + - --config-reloader-memory-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).memory) | default 0 }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.enableProbe }} + - --enable-config-reloader-probes=true + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceNamespaces }} + - --alertmanager-instance-namespaces={{ .Values.prometheusOperator.alertmanagerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceSelector }} + - --alertmanager-instance-selector={{ .Values.prometheusOperator.alertmanagerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerConfigNamespaces }} + - --alertmanager-config-namespaces={{ .Values.prometheusOperator.alertmanagerConfigNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceNamespaces }} + - --prometheus-instance-namespaces={{ .Values.prometheusOperator.prometheusInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceSelector }} + - --prometheus-instance-selector={{ .Values.prometheusOperator.prometheusInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.thanosImage.sha }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}@sha256:{{ .Values.prometheusOperator.thanosImage.sha }} + {{- else }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceNamespaces }} + - --thanos-ruler-instance-namespaces={{ .Values.prometheusOperator.thanosRulerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceSelector }} + - --thanos-ruler-instance-selector={{ .Values.prometheusOperator.thanosRulerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.secretFieldSelector }} + - --secret-field-selector={{ tpl (.Values.prometheusOperator.secretFieldSelector) $ }} + {{- end }} + {{- if .Values.prometheusOperator.clusterDomain }} + - --cluster-domain={{ .Values.prometheusOperator.clusterDomain }} + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - --web.enable-tls=true + - --web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }} + - --web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }} + - --web.listen-address=:{{ .Values.prometheusOperator.tls.internalPort }} + - --web.tls-min-version={{ .Values.prometheusOperator.tls.tlsMinVersion }} + ports: + - containerPort: {{ .Values.prometheusOperator.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + env: + {{- range $key, $value := .Values.prometheusOperator.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.containerSecurityContext | indent 12 }} + volumeMounts: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + mountPath: /cert + readOnly: true + {{- end }} + {{- with .Values.prometheusOperator.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + {{- end }} + {{- with .Values.prometheusOperator.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.prometheusOperator.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.tolerations }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/networkpolicy.yaml new file mode 100644 index 0000000000..cfd5b0b8c7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/networkpolicy.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + egress: + - {} + ingress: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort }} + {{- else }} + - port: 8080 + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml new file mode 100644 index 0000000000..61bc3d9040 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.operator.fullname" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..40e0fc5c15 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/psp.yaml new file mode 100644 index 0000000000..28a9075d3e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/psp.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: {{ .Values.prometheusOperator.hostNetwork }} + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/service.yaml new file mode 100644 index 0000000000..d45ab22d08 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/service.yaml @@ -0,0 +1,57 @@ +{{- if .Values.prometheusOperator.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.service.labels }} +{{ toYaml .Values.prometheusOperator.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/serviceaccount.yaml new file mode 100644 index 0000000000..4f84974f9b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +automountServiceAccountToken: {{ .Values.prometheusOperator.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/servicemonitor.yaml new file mode 100644 index 0000000000..cbe79e1253 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- with .Values.prometheusOperator.serviceMonitor.additionalLabels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheusOperator.serviceMonitor | nindent 2 }} + endpoints: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: https + scheme: https + tlsConfig: + serverName: {{ template "kube-prometheus-stack.operator.fullname" . }} + ca: + secret: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + key: {{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}ca.crt{{ else }}ca{{ end }} + optional: false + {{- else }} + - port: http + {{- end }} + honorLabels: true + {{- if .Values.prometheusOperator.serviceMonitor.interval }} + interval: {{ .Values.prometheusOperator.serviceMonitor.interval }} + {{- end }} + metricRelabelings: + {{- if .Values.prometheusOperator.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.prometheusOperator.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheusOperator.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheusOperator.serviceMonitor.relabelings | indent 6 }} +{{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..f225d16dde --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.prometheusOperator.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + {{- with .Values.prometheusOperator.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-prometheus-stack.name" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/_rules.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/_rules.tpl new file mode 100644 index 0000000000..4a8213d089 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/_rules.tpl @@ -0,0 +1,44 @@ +{{- /* +Generated file. Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- define "rules.names" }} +rules: + - "alertmanager.rules" + - "config-reloaders" + - "etcd" + - "general.rules" + - "k8s.rules.container-cpu-usage-seconds-total" + - "k8s.rules.container-memory-cache" + - "k8s.rules.container-memory-rss" + - "k8s.rules.container-memory-swap" + - "k8s.rules.container-memory-working-set-bytes" + - "k8s.rules.container-resource" + - "k8s.rules.pod-owner" + - "kube-apiserver-availability.rules" + - "kube-apiserver-burnrate.rules" + - "kube-apiserver-histogram.rules" + - "kube-apiserver-slos" + - "kube-prometheus-general.rules" + - "kube-prometheus-node-recording.rules" + - "kube-scheduler.rules" + - "kube-state-metrics" + - "kubelet.rules" + - "kubernetes-apps" + - "kubernetes-resources" + - "kubernetes-storage" + - "kubernetes-system" + - "kubernetes-system-kube-proxy" + - "kubernetes-system-apiserver" + - "kubernetes-system-kubelet" + - "kubernetes-system-controller-manager" + - "kubernetes-system-scheduler" + - "node-exporter.rules" + - "node-exporter" + - "node.rules" + - "node-network" + - "prometheus-operator" + - "prometheus" + - "windows.node.rules" + - "windows.pod.rules" +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml new file mode 100644 index 0000000000..bff930981a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-relabel-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alert-relabel-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml new file mode 100644 index 0000000000..2fe8fdb816 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alertmanager-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs) . | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalPrometheusRules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalPrometheusRules.yaml new file mode 100644 index 0000000000..cb4aabaa7b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalPrometheusRules.yaml @@ -0,0 +1,43 @@ +{{- if or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-additional-prometheus-rules + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- if .Values.additionalPrometheusRulesMap }} +{{- range $prometheusRuleName, $prometheusRule := .Values.additionalPrometheusRulesMap }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ $prometheusRuleName }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $prometheusRule.additionalLabels }} +{{ toYaml $prometheusRule.additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml $prometheusRule.groups| indent 8 }} +{{- end }} +{{- else }} +{{- range .Values.additionalPrometheusRules }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml .groups| indent 8 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml new file mode 100644 index 0000000000..ebdf766fde --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-scrape-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if eq ( typeOf .Values.prometheus.prometheusSpec.additionalScrapeConfigs ) "string" }} + additional-scrape-configs.yaml: {{ tpl .Values.prometheus.prometheusSpec.additionalScrapeConfigs $ | b64enc | quote }} +{{- else }} + additional-scrape-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalScrapeConfigs) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..74d61d7c13 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} +spec: + endpointSelector: + {{- if .Values.prometheus.networkPolicy.cilium.endpointSelector }} + {{- toYaml .Values.prometheus.networkPolicy.cilium.endpointSelector | nindent 4 }} + {{- else }} + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.egress }} + egress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.egress | nindent 4 }} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.ingress }} + ingress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.ingress | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/clusterrole.yaml new file mode 100644 index 0000000000..3585b5db11 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/clusterrole.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +# This permission are not in the kube-prometheus repo +# they're grabbed from https://github.com/prometheus/prometheus/blob/master/documentation/examples/rbac-setup.yml +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - "networking.k8s.io" + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +{{- if .Values.prometheus.additionalRulesForClusterRole }} +{{ toYaml .Values.prometheus.additionalRulesForClusterRole | indent 0 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/clusterrolebinding.yaml new file mode 100644 index 0000000000..9fc4f65da4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/csi-secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/csi-secret.yaml new file mode 100644 index 0000000000..e05382f633 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/csi-secret.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.secretProviderClass }} +--- +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +spec: +{{ toYaml .Values.prometheus.prometheusSpec.thanos.secretProviderClass | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/extrasecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/extrasecret.yaml new file mode 100644 index 0000000000..17f3478a46 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.prometheus.extraSecret.data -}} +{{- $secretName := printf "prometheus-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.prometheus.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.extraSecret.annotations }} + annotations: +{{ toYaml .Values.prometheus.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.prometheus.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ingress.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ingress.yaml new file mode 100644 index 0000000000..d2f6af5dd1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}} + {{- $pathType := .Values.prometheus.ingress.pathType | default "ImplementationSpecific" -}} + {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" -}} + {{- $servicePort := .Values.prometheus.ingress.servicePort | default .Values.prometheus.service.port -}} + {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} + {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} + {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} + {{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.ingress.annotations) . | nindent 4 }} +{{- end }} + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.ingress.labels }} +{{ toYaml .Values.prometheus.ingress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.ingress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.ingress.hosts }} + {{- range $host := .Values.prometheus.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.ingress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ingressThanosSidecar.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ingressThanosSidecar.yaml new file mode 100644 index 0000000000..3f507cfa9f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ingressThanosSidecar.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosIngress.enabled }} +{{- $pathType := .Values.prometheus.thanosIngress.pathType | default "" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "thanos-discovery" }} +{{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} +{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} +{{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.thanosIngress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.thanosIngress.annotations) . | nindent 4 }} +{{- end }} + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-gateway + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosIngress.labels }} +{{ toYaml .Values.prometheus.thanosIngress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.thanosIngress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.thanosIngress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.thanosIngress.hosts }} + {{- range $host := .Values.prometheus.thanosIngress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.thanosIngress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.thanosIngress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ingressperreplica.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ingressperreplica.yaml new file mode 100644 index 0000000000..1d76d135c8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled .Values.prometheus.ingressPerReplica.enabled }} +{{- $pathType := .Values.prometheus.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $servicePort := .Values.prometheus.servicePerReplica.port -}} +{{- $ingressValues := .Values.prometheus.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" $ }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/networkpolicy.yaml new file mode 100644 index 0000000000..1296a79063 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/networkpolicy.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + {{- if .Values.prometheus.networkPolicy.egress }} + egress: + {{- toYaml .Values.prometheus.networkPolicy.egress | nindent 4 }} + {{- end }} + {{- if .Values.prometheus.networkPolicy.ingress }} + ingress: + {{- toYaml .Values.prometheus.networkPolicy.ingress | nindent 4 }} + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + {{- if .Values.prometheus.networkPolicy.podSelector }} + {{- toYaml .Values.prometheus.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/nginx-config.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/nginx-config.yaml new file mode 100644 index 0000000000..e4d91f9a9e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/nginx-config.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-nginx-proxy-config + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + server { + listen 8081; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:9090/; + + sub_filter_once off; + sub_filter 'var PATH_PREFIX = "";' 'var PATH_PREFIX = ".";'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/podDisruptionBudget.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/podDisruptionBudget.yaml new file mode 100644 index 0000000000..48f3f1f5a6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/podDisruptionBudget.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.prometheus.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.prometheus.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.prometheus.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.prometheus.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/podmonitors.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/podmonitors.yaml new file mode 100644 index 0000000000..4e748c23b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/podmonitors.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalPodMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalPodMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + podMetricsEndpoints: +{{ toYaml .podMetricsEndpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .sampleLimit }} + sampleLimit: {{ .sampleLimit }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/prometheus.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/prometheus.yaml new file mode 100644 index 0000000000..5c3c8d4d1f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/prometheus.yaml @@ -0,0 +1,472 @@ +{{- if .Values.prometheus.enabled }} +{{- if .Values.prometheus.agentMode }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: PrometheusAgent +{{- else }} +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +{{- end }} +metadata: + name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +spec: +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.alertingEndpoints .Values.alertmanager.enabled) }} + alerting: + alertmanagers: +{{- if .Values.prometheus.prometheusSpec.alertingEndpoints }} +{{ toYaml .Values.prometheus.prometheusSpec.alertingEndpoints | indent 6 }} +{{- else if .Values.alertmanager.enabled }} + - namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + pathPrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.scheme }} + scheme: {{ .Values.alertmanager.alertmanagerSpec.scheme }} + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.tlsConfig }} + tlsConfig: +{{ toYaml .Values.alertmanager.alertmanagerSpec.tlsConfig | indent 10 }} + {{- end }} + apiVersion: {{ .Values.alertmanager.apiVersion }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.apiserverConfig }} + apiserverConfig: +{{ toYaml .Values.prometheus.prometheusSpec.apiserverConfig | indent 4}} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.prometheus.prometheusSpec.image.registry -}} + {{- if and .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}" + {{- end }} + version: {{ default .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.version }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.prometheus.prometheusSpec.additionalArgs | indent 4}} +{{- end -}} +{{- if .Values.prometheus.prometheusSpec.externalLabels }} + externalLabels: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusExternalLabelNameClear }} + prometheusExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.prometheusExternalLabelName }} + prometheusExternalLabelName: "{{ .Values.prometheus.prometheusSpec.prometheusExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.replicaExternalLabelNameClear }} + replicaExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.replicaExternalLabelName }} + replicaExternalLabelName: "{{ .Values.prometheus.prometheusSpec.replicaExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} + enableRemoteWriteReceiver: {{ .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalUrl }} + externalUrl: "{{ tpl .Values.prometheus.prometheusSpec.externalUrl . }}" +{{- else if and .Values.prometheus.ingress.enabled .Values.prometheus.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.prometheus.ingress.hosts 0) . }}{{ .Values.prometheus.prometheusSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "kube-prometheus-stack.namespace" . }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.nodeSelector }} +{{ toYaml .Values.prometheus.prometheusSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.prometheus.prometheusSpec.paused }} + replicas: {{ .Values.prometheus.prometheusSpec.replicas }} + shards: {{ .Values.prometheus.prometheusSpec.shards }} + logLevel: {{ .Values.prometheus.prometheusSpec.logLevel }} + logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} + listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} +{{- if not .Values.prometheus.agentMode }} + enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.web }} + web: +{{ toYaml .Values.prometheus.prometheusSpec.web | indent 4 }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.exemplars }} + exemplars: + {{ toYaml .Values.prometheus.prometheusSpec.exemplars | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableFeatures }} + enableFeatures: +{{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }} + - {{ tpl $enableFeatures $ }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeInterval }} + scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.prometheusSpec.scrapeTimeout }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.evaluationInterval }} + evaluationInterval: {{ .Values.prometheus.prometheusSpec.evaluationInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.resources }} + resources: +{{ toYaml .Values.prometheus.prometheusSpec.resources | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} + retention: {{ .Values.prometheus.prometheusSpec.retention | quote }} +{{- if .Values.prometheus.prometheusSpec.retentionSize }} + retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tsdb }} + tsdb: + {{- if .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + outOfOrderTimeWindow: {{ .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + {{- end }} +{{- end }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.walCompression false }} + walCompression: false +{{ else }} + walCompression: true +{{- end }} +{{- if .Values.prometheus.prometheusSpec.routePrefix }} + routePrefix: {{ .Values.prometheus.prometheusSpec.routePrefix | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.secrets }} + secrets: +{{ toYaml .Values.prometheus.prometheusSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.configMaps }} + configMaps: +{{ toYaml .Values.prometheus.prometheusSpec.configMaps | indent 4 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorSelector }} + serviceMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues }} + serviceMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + serviceMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector }} + serviceMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector | indent 4) . }} +{{ else }} + serviceMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorSelector }} + podMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues }} + podMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + podMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector }} + podMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector | indent 4) . }} +{{ else }} + podMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeSelector }} + probeSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.probeSelectorNilUsesHelmValues }} + probeSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + probeSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeNamespaceSelector }} + probeNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeNamespaceSelector | indent 4) . }} +{{ else }} + probeNamespaceSelector: {} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.remoteRead .Values.prometheus.prometheusSpec.additionalRemoteRead) }} + remoteRead: +{{- if .Values.prometheus.prometheusSpec.remoteRead }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteRead | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteRead }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteRead | indent 4 }} +{{- end }} +{{- end }} +{{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite) }} + remoteWrite: +{{- if .Values.prometheus.prometheusSpec.remoteWrite }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteWrite }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteWrite | indent 4 }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.prometheusSpec.securityContext | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleSelector | indent 4) . }} +{{- else if .Values.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigSelector }} + scrapeConfigSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.scrapeConfigSelectorNilUsesHelmValues }} + scrapeConfigSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + scrapeConfigSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector }} + scrapeConfigNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector | indent 4) . }} +{{ else }} + scrapeConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.storageSpec }} + storage: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.storageSpec | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMetadata }} + podMetadata: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMetadata | indent 4) . }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.query }} + query: +{{ toYaml .Values.prometheus.prometheusSpec.query | indent 4}} +{{- end }} +{{- if or .Values.prometheus.prometheusSpec.podAntiAffinity .Values.prometheus.prometheusSpec.affinity }} + affinity: +{{- if .Values.prometheus.prometheusSpec.affinity }} +{{ toYaml .Values.prometheus.prometheusSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.tolerations }} +{{ toYaml .Values.prometheus.prometheusSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.prometheus.prometheusSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} + additionalScrapeConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + key: additional-scrape-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.enabled }} + additionalScrapeConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.key }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if or .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + additionalAlertManagerConfigs: +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + key: additional-alertmanager-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + name: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.key }} + {{- if hasKey .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret "optional" }} + optional: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.optional }} + {{- end }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} + additionalAlertRelabelConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + key: additional-alert-relabel-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret }} + additionalAlertRelabelConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.key }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.containers }} + containers: +{{ tpl .Values.prometheus.prometheusSpec.containers $ | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.initContainers }} + initContainers: +{{ toYaml .Values.prometheus.prometheusSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.priorityClassName }} + priorityClassName: {{ .Values.prometheus.prometheusSpec.priorityClassName }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.thanos }} + thanos: +{{- with (omit .Values.prometheus.prometheusSpec.thanos "objectStorageConfig")}} +{{ toYaml . | indent 4 }} +{{- end }} +{{- if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).existingSecret) }} + objectStorageConfig: + key: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.name }}" +{{- else if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).secret) }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.disableCompaction }} + disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }} +{{- end }} +{{- end }} + portName: {{ .Values.prometheus.prometheusSpec.portName }} +{{- if .Values.prometheus.prometheusSpec.volumes }} + volumes: +{{ toYaml .Values.prometheus.prometheusSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.prometheus.prometheusSpec.volumeMounts | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs }} + arbitraryFSAccessThroughSMs: +{{ toYaml .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorLabels }} + overrideHonorLabels: {{ .Values.prometheus.prometheusSpec.overrideHonorLabels }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} + overrideHonorTimestamps: {{ .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + ignoreNamespaceSelectors: {{ .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} + enforcedNamespaceLabel: {{ .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} +{{- $prometheusDefaultRulesExcludedFromEnforce := (include "rules.names" .) | fromYaml }} +{{- if not .Values.prometheus.agentMode }} + prometheusRulesExcludedFromEnforce: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - ruleNamespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + ruleName: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce }} +{{ toYaml .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce | indent 4 }} +{{- end }} +{{- end }} + excludedFromEnforcement: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - group: monitoring.coreos.com + resource: prometheusrules + namespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + name: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.excludedFromEnforcement }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.excludedFromEnforcement | indent 4) . }} +{{- end }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.queryLogFile }} + queryLogFile: {{ .Values.prometheus.prometheusSpec.queryLogFile }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.sampleLimit }} + sampleLimit: {{ .Values.prometheus.prometheusSpec.sampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} + enforcedKeepDroppedTargets: {{ .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedSampleLimit }} + enforcedSampleLimit: {{ .Values.prometheus.prometheusSpec.enforcedSampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedTargetLimit }} + enforcedTargetLimit: {{ .Values.prometheus.prometheusSpec.enforcedTargetLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelLimit }} + enforcedLabelLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} + enforcedLabelNameLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit}} + enforcedLabelValueLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} + allowOverlappingBlocks: {{ .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.minReadySeconds }} + minReadySeconds: {{ .Values.prometheus.prometheusSpec.minReadySeconds }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} + maximumStartupDurationSeconds: {{ .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} +{{- end }} + hostNetwork: {{ .Values.prometheus.prometheusSpec.hostNetwork }} +{{- if .Values.prometheus.prometheusSpec.hostAliases }} + hostAliases: +{{ toYaml .Values.prometheus.prometheusSpec.hostAliases | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tracingConfig }} + tracingConfig: +{{ toYaml .Values.prometheus.prometheusSpec.tracingConfig | indent 4 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/psp-clusterrole.yaml new file mode 100644 index 0000000000..71476cd18b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/psp-clusterrole.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..a393928c78 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/psp.yaml new file mode 100644 index 0000000000..62d3854151 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/psp.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' +{{- if .Values.prometheus.podSecurityPolicy.volumes }} +{{ toYaml .Values.prometheus.podSecurityPolicy.volumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- if .Values.prometheus.podSecurityPolicy.allowedCapabilities }} + allowedCapabilities: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedCapabilities | indent 4 }} +{{- end }} +{{- if .Values.prometheus.podSecurityPolicy.allowedHostPaths }} + allowedHostPaths: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedHostPaths | indent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml new file mode 100644 index 0000000000..b66f052ade --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -0,0 +1,305 @@ +{{- /* +Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/alertmanager-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} +{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: alertmanager.rules + rules: +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedReload | default false) }} + - alert: AlertmanagerFailedReload + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Configuration has failed to load for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedreload + summary: Reloading an Alertmanager configuration has failed. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "AlertmanagerFailedReload" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedReload" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerMembersInconsistent | default false) }} + - alert: AlertmanagerMembersInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} has only found {{`{{`}} $value {{`}}`}} members of the {{`{{`}}$labels.job{{`}}`}} cluster. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagermembersinconsistent + summary: A member of an Alertmanager cluster has not found all other cluster members. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + < on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) group_left + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) (max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m])) + for: {{ dig "AlertmanagerMembersInconsistent" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerMembersInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedToSendAlerts | default false) }} + - alert: AlertmanagerFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} failed to send {{`{{`}} $value | humanizePercentage {{`}}`}} of notifications to {{`{{`}} $labels.integration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedtosendalerts + summary: An Alertmanager instance failed to send notifications. + expr: |- + ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerConfigInconsistent | default false) }} + - alert: AlertmanagerConfigInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have different configurations. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerconfiginconsistent + summary: Alertmanager instances within the same cluster have different configurations. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + count_values by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + ) + != 1 + for: {{ dig "AlertmanagerConfigInconsistent" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerConfigInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterDown | default false) }} + - alert: AlertmanagerClusterDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have been up for less than half of the last 5m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterdown + summary: Half or more of the Alertmanager instances within the same cluster are down. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + avg_over_time(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) < 0.5 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterCrashlooping | default false) }} + - alert: AlertmanagerClusterCrashlooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have restarted at least 5 times in the last 10m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclustercrashlooping + summary: Half or more of the Alertmanager instances within the same cluster are crashlooping. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + changes(process_start_time_seconds{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[10m]) > 4 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterCrashlooping" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterCrashlooping" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml new file mode 100644 index 0000000000..8416d6df40 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml @@ -0,0 +1,57 @@ +{{- /* +Generated from 'config-reloaders' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.configReloaders }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "config-reloaders" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: config-reloaders + rules: +{{- if not (.Values.defaultRules.disabled.ConfigReloaderSidecarErrors | default false) }} + - alert: ConfigReloaderSidecarErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders | indent 8 }} +{{- end }} + description: 'Errors encountered while the {{`{{`}}$labels.pod{{`}}`}} config-reloader sidecar attempts to sync config in {{`{{`}}$labels.namespace{{`}}`}} namespace. + + As a result, configuration for service running in {{`{{`}}$labels.pod{{`}}`}} may be stale and cannot be updated anymore.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/configreloadersidecarerrors + summary: config-reloader sidecar has not had a successful reload for 10m + expr: max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 + for: {{ dig "ConfigReloaderSidecarErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "ConfigReloaderSidecarErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/etcd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/etcd.yaml new file mode 100644 index 0000000000..a1d7a508f8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/etcd.yaml @@ -0,0 +1,461 @@ +{{- /* +Generated from 'etcd' group from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.etcd }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: etcd + rules: +{{- if not (.Values.defaultRules.disabled.etcdMembersDown | default false) }} + - alert: etcdMembersDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster members are down. + expr: |- + max without (endpoint) ( + sum without (instance) (up{job=~".*etcd.*"} == bool 0) + or + count without (To) ( + sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + ) + ) + > 0 + for: {{ dig "etcdMembersDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMembersDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdInsufficientMembers | default false) }} + - alert: etcdInsufficientMembers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster has insufficient number of members. + expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2) + for: {{ dig "etcdInsufficientMembers" "for" "3m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdInsufficientMembers" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdNoLeader | default false) }} + - alert: etcdNoLeader + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.' + summary: etcd cluster has no leader. + expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 + for: {{ dig "etcdNoLeader" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdNoLeader" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfLeaderChanges | default false) }} + - alert: etcdHighNumberOfLeaderChanges + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' + summary: etcd cluster has high number of leader changes. + expr: increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4 + for: {{ dig "etcdHighNumberOfLeaderChanges" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfLeaderChanges" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 1 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 5 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdGRPCRequestsSlow | default false) }} + - alert: etcdGRPCRequestsSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.' + summary: etcd grpc requests are slow + expr: |- + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) + > 0.15 + for: {{ dig "etcdGRPCRequestsSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdGRPCRequestsSlow" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdMemberCommunicationSlow | default false) }} + - alert: etcdMemberCommunicationSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster member communication is slow. + expr: |- + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.15 + for: {{ dig "etcdMemberCommunicationSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMemberCommunicationSlow" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedProposals | default false) }} + - alert: etcdHighNumberOfFailedProposals + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of proposal failures. + expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + for: {{ dig "etcdHighNumberOfFailedProposals" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedProposals" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.5 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 1 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighCommitDurations | default false) }} + - alert: etcdHighCommitDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile commit durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.25 + for: {{ dig "etcdHighCommitDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighCommitDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseQuotaLowSpace | default false) }} + - alert: etcdDatabaseQuotaLowSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' + summary: etcd cluster database is running full. + expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95 + for: {{ dig "etcdDatabaseQuotaLowSpace" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseQuotaLowSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdExcessiveDatabaseGrowth | default false) }} + - alert: etcdExcessiveDatabaseGrowth + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.' + summary: etcd cluster database growing very fast. + expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"} + for: {{ dig "etcdExcessiveDatabaseGrowth" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdExcessiveDatabaseGrowth" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseHighFragmentationRatio | default false) }} + - alert: etcdDatabaseHighFragmentationRatio + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size in use on instance {{`{{`}} $labels.instance {{`}}`}} is {{`{{`}} $value | humanizePercentage {{`}}`}} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.' + runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation + summary: etcd database size in use is less than 50% of the actual allocated storage. + expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600 + for: {{ dig "etcdDatabaseHighFragmentationRatio" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseHighFragmentationRatio" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml new file mode 100644 index 0000000000..8aca0b85f5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml @@ -0,0 +1,125 @@ +{{- /* +Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: general.rules + rules: +{{- if not (.Values.defaultRules.disabled.TargetDown | default false) }} + - alert: TargetDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/targetdown + summary: One or more targets are unreachable. + expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10 + for: {{ dig "TargetDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "TargetDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.Watchdog | default false) }} + - alert: Watchdog + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + + This alert is always firing, therefore it should always be firing in Alertmanager + + and always fire against a receiver. There are integrations with various notification + + mechanisms that send a notification when this alert is not firing. For example the + + "DeadMansSnitch" integration in PagerDuty. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager is working properly. + expr: vector(1) + labels: + severity: {{ dig "Watchdog" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.InfoInhibitor | default false) }} + - alert: InfoInhibitor + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert that is used to inhibit info alerts. + + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + + other alerts. + + This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a + + severity of ''warning'' or ''critical'' starts firing on the same namespace. + + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: {{ dig "InfoInhibitor" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml new file mode 100644 index 0000000000..9de5f5bc9c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml @@ -0,0 +1,43 @@ +{{- /* +Generated from 'k8s.rules.container-cpu-usage-seconds-total' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerCpuUsageSecondsTotal }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-cpu-usage-seconds-total" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_cpu_usage_seconds_total + rules: + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml new file mode 100644 index 0000000000..323f41f9cb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-cache' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryCache }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-cache" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_cache + rules: + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml new file mode 100644 index 0000000000..312d73c889 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-rss' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryRss }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-rss" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_rss + rules: + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml new file mode 100644 index 0000000000..136595e801 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-swap' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemorySwap }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-swap" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_swap + rules: + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml new file mode 100644 index 0000000000..d308b7473a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-working-set-bytes' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryWorkingSetBytes }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-working-set-bytes" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_working_set_bytes + rules: + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml new file mode 100644 index 0000000000..2d896e59e4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml @@ -0,0 +1,168 @@ +{{- /* +Generated from 'k8s.rules.container-resource' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerResource }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-resource" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_resource + rules: + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml new file mode 100644 index 0000000000..4915b25e73 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml @@ -0,0 +1,107 @@ +{{- /* +Generated from 'k8s.rules.pod-owner' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sPodOwner }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.pod-owner" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.pod_owner + rules: + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) group_left(owner_name) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="{{ $kubeStateMetricsJob }}"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml new file mode 100644 index 0000000000..c61bd222ab --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -0,0 +1,237 @@ +{{- /* +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8s }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules + rules: + - expr: |- + sum by (cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( + 1, max by (replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml new file mode 100644 index 0000000000..6194e9c614 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -0,0 +1,273 @@ +{{- /* +Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverAvailability }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-availability.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - interval: 3m + name: kube-apiserver-availability.rules + rules: + - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30 + record: code_verb:apiserver_request_total:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (increase(apiserver_request_sli_duration_seconds_count{job="apiserver"}[1h])) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h])) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + # write too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + ( + # read too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + ) + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d) + labels: + verb: all + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + # too slow + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + ( + # too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml new file mode 100644 index 0000000000..e6666a6f41 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml @@ -0,0 +1,440 @@ +{{- /* +Generated from 'kube-apiserver-burnrate.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverBurnrate }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-burnrate.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-burnrate.rules + rules: + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml new file mode 100644 index 0000000000..d145341952 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml @@ -0,0 +1,53 @@ +{{- /* +Generated from 'kube-apiserver-histogram.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverHistogram }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-histogram.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-histogram.rules + rules: + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml new file mode 100644 index 0000000000..30ef9a4293 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -0,0 +1,159 @@ +{{- /* +Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverSlos }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-slos" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-slos + rules: +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) + and + sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) + and + sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 6h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 30m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) + and + sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 2h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) + and + sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "3h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 3d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 6h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml new file mode 100644 index 0000000000..fcf35f389b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml @@ -0,0 +1,49 @@ +{{- /* +Generated from 'kube-prometheus-general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusGeneral }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-general.rules + rules: + - expr: count without(instance, pod, node) (up == 1) + record: count:up1 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: count without(instance, pod, node) (up == 0) + record: count:up0 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml new file mode 100644 index 0000000000..7a0d202324 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml @@ -0,0 +1,93 @@ +{{- /* +Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-recording.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-node-recording.rules + rules: + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance) + record: instance:node_cpu:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) + record: instance:node_network_receive_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) + record: instance:node_network_transmit_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) + record: instance:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) + record: cluster:node_cpu:sum_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) + record: cluster:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml new file mode 100644 index 0000000000..c9d61ce37b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml @@ -0,0 +1,135 @@ +{{- /* +Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-scheduler.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-scheduler.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml new file mode 100644 index 0000000000..d1ad3cae5e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml @@ -0,0 +1,152 @@ +{{- /* +Generated from 'kube-state-metrics' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubeStateMetrics-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeStateMetrics }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-state-metrics" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-state-metrics + rules: +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsListErrors | default false) }} + - alert: KubeStateMetricsListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricslisterrors + summary: kube-state-metrics is experiencing errors in list operations. + expr: |- + (sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsListErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsWatchErrors | default false) }} + - alert: KubeStateMetricsWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricswatcherrors + summary: kube-state-metrics is experiencing errors in watch operations. + expr: |- + (sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsWatchErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardingMismatch | default false) }} + - alert: KubeStateMetricsShardingMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardingmismatch + summary: kube-state-metrics sharding is misconfigured. + expr: stdvar (kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) != 0 + for: {{ dig "KubeStateMetricsShardingMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardingMismatch" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardsMissing | default false) }} + - alert: KubeStateMetricsShardsMissing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardsmissing + summary: kube-state-metrics shards are missing. + expr: |- + 2^max(kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - 1 + - + sum( 2 ^ max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="{{ $kubeStateMetricsJob }}"}) ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + != 0 + for: {{ dig "KubeStateMetricsShardsMissing" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardsMissing" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml new file mode 100644 index 0000000000..39fdddf3fe --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml @@ -0,0 +1,65 @@ +{{- /* +Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubelet }} +{{- if (include "exporter.kubelet.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubelet.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubelet.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml new file mode 100644 index 0000000000..2a861a522c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -0,0 +1,568 @@ +{{- /* +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesApps }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-apps + rules: +{{- if not (.Values.defaultRules.disabled.KubePodCrashLooping | default false) }} + - alert: KubePodCrashLooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: 'Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is in waiting state (reason: "CrashLoopBackOff").' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodcrashlooping + summary: Pod is crash looping. + expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) >= 1 + for: {{ dig "KubePodCrashLooping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodCrashLooping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePodNotReady | default false) }} + - alert: KubePodNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodnotready + summary: Pod has been in a non-ready state for more than 15 minutes. + expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown|Failed"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left(owner_kind) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + for: {{ dig "KubePodNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentGenerationMismatch | default false) }} + - alert: KubeDeploymentGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentgenerationmismatch + summary: Deployment generation mismatch due to possible roll-back + expr: |- + kube_deployment_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeDeploymentGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentReplicasMismatch | default false) }} + - alert: KubeDeploymentReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_deployment_spec_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_deployment_status_replicas_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeDeploymentReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentRolloutStuck | default false) }} + - alert: KubeDeploymentRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Rollout of deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} is not progressing for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentrolloutstuck + summary: Deployment rollout is not progressing. + expr: |- + kube_deployment_status_condition{condition="Progressing", status="false",job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != 0 + for: {{ dig "KubeDeploymentRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetReplicasMismatch | default false) }} + - alert: KubeStatefulSetReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetreplicasmismatch + summary: StatefulSet has not matched the expected number of replicas. + expr: |- + ( + kube_statefulset_status_replicas_ready{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetGenerationMismatch | default false) }} + - alert: KubeStatefulSetGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetgenerationmismatch + summary: StatefulSet generation mismatch due to possible roll-back + expr: |- + kube_statefulset_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeStatefulSetGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetUpdateNotRolledOut | default false) }} + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetupdatenotrolledout + summary: StatefulSet update has not been rolled out. + expr: |- + ( + max without (revision) ( + kube_statefulset_status_current_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetUpdateNotRolledOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetUpdateNotRolledOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetRolloutStuck | default false) }} + - alert: KubeDaemonSetRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} has not finished or progressed for at least 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetrolloutstuck + summary: DaemonSet rollout is stuck. + expr: |- + ( + ( + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + 0 + ) or ( + kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeDaemonSetRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeContainerWaiting | default false) }} + - alert: KubeContainerWaiting + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontainerwaiting + summary: Pod container waiting longer than 1 hour + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) > 0 + for: {{ dig "KubeContainerWaiting" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeContainerWaiting" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetNotScheduled | default false) }} + - alert: KubeDaemonSetNotScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetnotscheduled + summary: DaemonSet pods are not scheduled. + expr: |- + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + - + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetNotScheduled" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetNotScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetMisScheduled | default false) }} + - alert: KubeDaemonSetMisScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetmisscheduled + summary: DaemonSet pods are misscheduled. + expr: kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetMisScheduled" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetMisScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobNotCompleted | default false) }} + - alert: KubeJobNotCompleted + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than {{`{{`}} "43200" | humanizeDuration {{`}}`}} to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobnotcompleted + summary: Job did not complete in time + expr: |- + time() - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, job_name, cluster) (kube_job_status_start_time{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + and + kube_job_status_active{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0) > 43200 + labels: + severity: {{ dig "KubeJobNotCompleted" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobFailed | default false) }} + - alert: KubeJobFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. Removing failed job after investigation should clear this alert. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobfailed + summary: Job failed to complete. + expr: kube_job_failed{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeJobFailed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeJobFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaReplicasMismatch | default false) }} + - alert: KubeHpaReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpareplicasmismatch + summary: HPA has not matched desired number of replicas. + expr: |- + (kube_horizontalpodautoscaler_status_desired_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_horizontalpodautoscaler_spec_min_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + < + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + changes(kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 + for: {{ dig "KubeHpaReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaMaxedOut | default false) }} + - alert: KubeHpaMaxedOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has been running at max replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpamaxedout + summary: HPA is running at max replicas + expr: |- + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + == + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeHpaMaxedOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaMaxedOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml new file mode 100644 index 0000000000..1d32f9bbad --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -0,0 +1,282 @@ +{{- /* +Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesResources }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-resources" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-resources + rules: +{{- if not (.Values.defaultRules.disabled.KubeCPUOvercommit | default false) }} + - alert: KubeCPUOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Pods by {{`{{`}} $value {{`}}`}} CPU shares and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(namespace_cpu:kube_pod_container_resource_requests:sum{job="{{ $kubeStateMetricsJob }}",}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeCPUOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryOvercommit | default false) }} + - alert: KubeMemoryOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeMemoryOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeCPUQuotaOvercommit | default false) }} + - alert: KubeCPUQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuquotaovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(cpu|requests.cpu)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="cpu", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeCPUQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryQuotaOvercommit | default false) }} + - alert: KubeMemoryQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryquotaovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(memory|requests.memory)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeMemoryQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaAlmostFull | default false) }} + - alert: KubeQuotaAlmostFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaalmostfull + summary: Namespace quota is going to be full. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 0.9 < 1 + for: {{ dig "KubeQuotaAlmostFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaAlmostFull" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaFullyUsed | default false) }} + - alert: KubeQuotaFullyUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotafullyused + summary: Namespace quota is fully used. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + == 1 + for: {{ dig "KubeQuotaFullyUsed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaFullyUsed" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaExceeded | default false) }} + - alert: KubeQuotaExceeded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaexceeded + summary: Namespace quota has exceeded the limits. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 1 + for: {{ dig "KubeQuotaExceeded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaExceeded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.CPUThrottlingHigh | default false) }} + - alert: CPUThrottlingHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.pod {{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/cputhrottlinghigh + summary: Processes experience elevated CPU throttling. + expr: |- + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + / + sum(increase(container_cpu_cfs_periods_total{}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + > ( 25 / 100 ) + for: {{ dig "CPUThrottlingHigh" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "CPUThrottlingHigh" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml new file mode 100644 index 0000000000..b988445653 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -0,0 +1,216 @@ +{{- /* +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-storage + rules: +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + < 0.03 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} only has {{`{{`}} $value | humanizePercentage {{`}}`}} free inodes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to run out of inodes within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} of its inodes are free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeErrors | default false) }} + - alert: KubePersistentVolumeErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeerrors + summary: PersistentVolume is having issues with provisioning. + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="{{ $kubeStateMetricsJob }}"} > 0 + for: {{ dig "KubePersistentVolumeErrors" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml new file mode 100644 index 0000000000..af34a23f88 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -0,0 +1,193 @@ +{{- /* +Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-apiserver" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-apiserver + rules: +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIErrors | default false) }} + - alert: KubeAggregatedAPIErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapierrors + summary: Kubernetes aggregated API has reported errors. + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4 + labels: + severity: {{ dig "KubeAggregatedAPIErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIDown | default false) }} + - alert: KubeAggregatedAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapidown + summary: Kubernetes aggregated API is down. + expr: (1 - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85 + for: {{ dig "KubeAggregatedAPIDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAggregatedAPIDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if .Values.kubeApiServer.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeAPIDown | default false) }} + - alert: KubeAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: KubeAPI has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapidown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="apiserver"} == 1) + for: {{ dig "KubeAPIDown" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPIDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPITerminatedRequests | default false) }} + - alert: KubeAPITerminatedRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapiterminatedrequests + summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 + for: {{ dig "KubeAPITerminatedRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPITerminatedRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml new file mode 100644 index 0000000000..205bd59800 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeControllerManager }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-controller-manager" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-controller-manager + rules: +{{- if not (.Values.defaultRules.disabled.KubeControllerManagerDown | default false) }} + - alert: KubeControllerManagerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager | indent 8 }} +{{- end }} + description: KubeControllerManager has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontrollermanagerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeControllerManager.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeControllerManagerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} + diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml new file mode 100644 index 0000000000..66b1d62001 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml @@ -0,0 +1,56 @@ +{{- /* +Generated from 'kubernetes-system-kube-proxy' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeProxy }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kube-proxy" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kube-proxy + rules: +{{- if not (.Values.defaultRules.disabled.KubeProxyDown | default false) }} + - alert: KubeProxyDown + annotations: + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + description: KubeProxy has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeproxydown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeProxy.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeProxyDown" "labelsSeverity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml new file mode 100644 index 0000000000..2a55676735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -0,0 +1,379 @@ +{{- /* +Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kubelet" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kubelet + rules: +{{- if not (.Values.defaultRules.disabled.KubeNodeNotReady | default false) }} + - alert: KubeNodeNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than 15 minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodenotready + summary: Node is not ready. + expr: kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",condition="Ready",status="true"} == 0 + for: {{ dig "KubeNodeNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeUnreachable | default false) }} + - alert: KubeNodeUnreachable + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} is unreachable and some workloads may be rescheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodeunreachable + summary: Node is unreachable. + expr: (kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1 + for: {{ dig "KubeNodeUnreachable" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeUnreachable" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletTooManyPods | default false) }} + - alert: KubeletTooManyPods + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubelettoomanypods + summary: Kubelet is running at capacity. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + (kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}",phase="Running"} == 1) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) (1, kube_pod_info{job="{{ $kubeStateMetricsJob }}"}) + ) + / + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + kube_node_status_capacity{job="{{ $kubeStateMetricsJob }}",resource="pods"} != 1 + ) > 0.95 + for: {{ dig "KubeletTooManyPods" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletTooManyPods" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeReadinessFlapping | default false) }} + - alert: KubeNodeReadinessFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodereadinessflapping + summary: Node readiness status is flapping. + expr: sum(changes(kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",status="true",condition="Ready"}[15m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) > 2 + for: {{ dig "KubeNodeReadinessFlapping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeReadinessFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPlegDurationHigh | default false) }} + - alert: KubeletPlegDurationHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletplegdurationhigh + summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist. + expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10 + for: {{ dig "KubeletPlegDurationHigh" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletPlegDurationHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPodStartUpLatencyHigh | default false) }} + - alert: KubeletPodStartUpLatencyHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletpodstartuplatencyhigh + summary: Kubelet Pod startup latency is too high. + expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} > 60 + for: 15m + labels: + severity: {{ dig "KubeletPodStartUpLatencyHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateRenewalErrors | default false) }} + - alert: KubeletClientCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its client certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificaterenewalerrors + summary: Kubelet has failed to renew its client certificate. + expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletClientCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletClientCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateRenewalErrors | default false) }} + - alert: KubeletServerCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its server certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificaterenewalerrors + summary: Kubelet has failed to renew its server certificate. + expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletServerCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletServerCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if (include "exporter.kubelet.enabled" .)}} +{{- if not (.Values.defaultRules.disabled.KubeletDown | default false) }} + - alert: KubeletDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} == 1) + for: 15m + labels: + severity: {{ dig "KubeletDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml new file mode 100644 index 0000000000..9890b1c959 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -0,0 +1,54 @@ +{{- /* +Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-scheduler" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-scheduler + rules: +{{- if .Values.kubeScheduler.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeSchedulerDown | default false) }} + - alert: KubeSchedulerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting | indent 8 }} +{{- end }} + description: KubeScheduler has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeschedulerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeScheduler.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeSchedulerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml new file mode 100644 index 0000000000..621326d0ad --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -0,0 +1,87 @@ +{{- /* +Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system + rules: +{{- if not (.Values.defaultRules.disabled.KubeVersionMismatch | default false) }} + - alert: KubeVersionMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeversionmismatch + summary: Different semantic versions of Kubernetes components running. + expr: count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 + for: {{ dig "KubeVersionMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeVersionMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientErrors | default false) }} + - alert: KubeClientErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} $value | humanizePercentage {{`}}`}} errors.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclienterrors + summary: Kubernetes API server client is experiencing errors. + expr: |- + (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace) + / + sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace)) + > 0.01 + for: {{ dig "KubeClientErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml new file mode 100644 index 0000000000..5d4711ae08 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml @@ -0,0 +1,188 @@ +{{- /* +Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter.rules + rules: + - expr: |- + count without (cpu, mode) ( + node_cpu_seconds_total{job="node-exporter",mode="idle"} + ) + record: instance:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg without (cpu) ( + sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m])) + ) + record: instance:node_cpu_utilisation:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + ( + node_load1{job="node-exporter"} + / + instance:node_num_cpu:sum{job="node-exporter"} + ) + record: instance:node_load1_per_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + node_memory_MemAvailable_bytes{job="node-exporter"} + or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + + node_memory_Cached_bytes{job="node-exporter"} + + + node_memory_MemFree_bytes{job="node-exporter"} + + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) + / + node_memory_MemTotal_bytes{job="node-exporter"} + ) + record: instance:node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) + record: instance:node_vmstat_pgmajfault:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_weighted_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml new file mode 100644 index 0000000000..14738cedfa --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml @@ -0,0 +1,801 @@ +{{- /* +Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter + rules: +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 24 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 4 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 5% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 3% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 24 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 4 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 5% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 3% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkReceiveErrs | default false) }} + - alert: NodeNetworkReceiveErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworkreceiveerrs + summary: Network interface is reporting many receive errors. + expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkReceiveErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkReceiveErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkTransmitErrs | default false) }} + - alert: NodeNetworkTransmitErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworktransmiterrs + summary: Network interface is reporting many transmit errors. + expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkTransmitErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkTransmitErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeHighNumberConntrackEntriesUsed | default false) }} + - alert: NodeHighNumberConntrackEntriesUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodehighnumberconntrackentriesused + summary: Number of conntrack are getting close to the limit. + expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75 + labels: + severity: {{ dig "NodeHighNumberConntrackEntriesUsed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeTextFileCollectorScrapeError | default false) }} + - alert: NodeTextFileCollectorScrapeError + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodetextfilecollectorscrapeerror + summary: Node Exporter text file collector failed to scrape. + expr: node_textfile_scrape_error{job="node-exporter"} == 1 + labels: + severity: {{ dig "NodeTextFileCollectorScrapeError" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockSkewDetected | default false) }} + - alert: NodeClockSkewDetected + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclockskewdetected + summary: Clock skew detected. + expr: |- + ( + node_timex_offset_seconds{job="node-exporter"} > 0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0 + ) + or + ( + node_timex_offset_seconds{job="node-exporter"} < -0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0 + ) + for: {{ dig "NodeClockSkewDetected" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockSkewDetected" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockNotSynchronising | default false) }} + - alert: NodeClockNotSynchronising + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclocknotsynchronising + summary: Clock not synchronising. + expr: |- + min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0 + and + node_timex_maxerror_seconds{job="node-exporter"} >= 16 + for: {{ dig "NodeClockNotSynchronising" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockNotSynchronising" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDegraded | default false) }} + - alert: NodeRAIDDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: RAID array '{{`{{`}} $labels.device {{`}}`}}' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddegraded + summary: RAID Array is degraded. + expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0 + for: {{ dig "NodeRAIDDegraded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeRAIDDegraded" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDiskFailure | default false) }} + - alert: NodeRAIDDiskFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddiskfailure + summary: Failed device in RAID array. + expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0 + labels: + severity: {{ dig "NodeRAIDDiskFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeCPUHighUsage | default false) }} + - alert: NodeCPUHighUsage + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodecpuhighusage + summary: High CPU usage. + expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!="idle"}[2m]))) * 100 > 90 + for: {{ dig "NodeCPUHighUsage" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeCPUHighUsage" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemSaturation | default false) }} + - alert: NodeSystemSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This might indicate this instance resources saturation and can cause it becoming unresponsive. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemsaturation + summary: System saturated, load per core is very high. + expr: |- + node_load1{job="node-exporter"} + / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2 + for: {{ dig "NodeSystemSaturation" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryMajorPagesFaults | default false) }} + - alert: NodeMemoryMajorPagesFaults + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + Please check that there is enough memory available at this instance. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememorymajorpagesfaults + summary: Memory major page faults are occurring at very high rate. + expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500 + for: {{ dig "NodeMemoryMajorPagesFaults" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryMajorPagesFaults" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryHighUtilization | default false) }} + - alert: NodeMemoryHighUtilization + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememoryhighutilization + summary: Host is running out of memory. + expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90 + for: {{ dig "NodeMemoryHighUtilization" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryHighUtilization" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeDiskIOSaturation | default false) }} + - alert: NodeDiskIOSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 30 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This symptom might indicate disk saturation. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodediskiosaturation + summary: Disk IO queue is high. + expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10 + for: {{ dig "NodeDiskIOSaturation" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeDiskIOSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemdServiceFailed | default false) }} + - alert: NodeSystemdServiceFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemdservicefailed + summary: Systemd service has entered failed state. + expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1 + for: {{ dig "NodeSystemdServiceFailed" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemdServiceFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeBondingDegraded | default false) }} + - alert: NodeBondingDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodebondingdegraded + summary: Bonding interface is degraded + expr: (node_bonding_slaves - node_bonding_active) != 0 + for: {{ dig "NodeBondingDegraded" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeBondingDegraded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node-network.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node-network.yaml new file mode 100644 index 0000000000..8dc60ef66b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node-network.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.network }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-network" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-network + rules: +{{- if not (.Values.defaultRules.disabled.NodeNetworkInterfaceFlapping | default false) }} + - alert: NodeNetworkInterfaceFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.network }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.network | indent 8 }} +{{- end }} + description: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing its up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/nodenetworkinterfaceflapping + summary: Network interface is often changing its status + expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 + for: {{ dig "NodeNetworkInterfaceFlapping" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkInterfaceFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml new file mode 100644 index 0000000000..e2911b905e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml @@ -0,0 +1,109 @@ +{{- /* +Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.node }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node.rules + rules: + - expr: |- + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node, namespace, pod) ( + label_replace(kube_pod_info{job="{{ $kubeStateMetricsJob }}",node!=""}, "pod", "$1", "pod", "(.*)") + )) + record: 'node_namespace_pod:kube_pod_info:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + node_cpu_seconds_total{mode="idle",job="node-exporter"} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, node_namespace_pod:kube_pod_info:) + ) + record: node:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum( + node_memory_MemAvailable_bytes{job="node-exporter"} or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + node_memory_Cached_bytes{job="node-exporter"} + + node_memory_MemFree_bytes{job="node-exporter"} + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + record: :node_memory_MemAvailable_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + sum without (mode) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m]) + ) + ) + record: node:node_cpu_utilization:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + node:node_cpu_utilization:ratio_rate5m + ) + record: cluster:node_cpu:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml new file mode 100644 index 0000000000..1f288dffe3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml @@ -0,0 +1,253 @@ +{{- /* +Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheusOperator }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus-operator" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus-operator + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorListErrors | default false) }} + - alert: PrometheusOperatorListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing List operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorlisterrors + summary: Errors while performing list operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m]))) > 0.4 + for: {{ dig "PrometheusOperatorListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorListErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorWatchErrors | default false) }} + - alert: PrometheusOperatorWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing watch operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorwatcherrors + summary: Errors while performing watch operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.4 + for: {{ dig "PrometheusOperatorWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorWatchErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorSyncFailed | default false) }} + - alert: PrometheusOperatorSyncFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Controller {{`{{`}} $labels.controller {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} namespace fails to reconcile {{`{{`}} $value {{`}}`}} objects. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorsyncfailed + summary: Last controller reconciliation failed + expr: min_over_time(prometheus_operator_syncs{status="failed",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorSyncFailed" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorSyncFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorReconcileErrors | default false) }} + - alert: PrometheusOperatorReconcileErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of reconciling operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorreconcileerrors + summary: Errors while reconciling objects. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorReconcileErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorReconcileErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorStatusUpdateErrors | default false) }} + - alert: PrometheusOperatorStatusUpdateErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of status update operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorstatusupdateerrors + summary: Errors while updating objects status. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorStatusUpdateErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorStatusUpdateErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNodeLookupErrors | default false) }} + - alert: PrometheusOperatorNodeLookupErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while reconciling Prometheus in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornodelookuperrors + summary: Errors while reconciling Prometheus. + expr: rate(prometheus_operator_node_address_lookup_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: {{ dig "PrometheusOperatorNodeLookupErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNodeLookupErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNotReady | default false) }} + - alert: PrometheusOperatorNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace isn't ready to reconcile {{`{{`}} $labels.controller {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornotready + summary: Prometheus operator not ready + expr: min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) == 0) + for: {{ dig "PrometheusOperatorNotReady" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorRejectedResources | default false) }} + - alert: PrometheusOperatorRejectedResources + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace rejected {{`{{`}} printf "%0.0f" $value {{`}}`}} {{`{{`}} $labels.controller {{`}}`}}/{{`{{`}} $labels.resource {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorrejectedresources + summary: Resources rejected by Prometheus operator + expr: min_over_time(prometheus_operator_managed_resources{state="rejected",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorRejectedResources" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorRejectedResources" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml new file mode 100644 index 0000000000..9dfeb1f9db --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml @@ -0,0 +1,707 @@ +{{- /* +Generated from 'prometheus' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} +{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusBadConfig | default false) }} + - alert: PrometheusBadConfig + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to reload its configuration. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusbadconfig + summary: Failed Prometheus configuration reload. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "PrometheusBadConfig" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusBadConfig" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusSDRefreshFailure | default false) }} + - alert: PrometheusSDRefreshFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to refresh SD with mechanism {{`{{`}}$labels.mechanism{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheussdrefreshfailure + summary: Failed Prometheus SD refresh. + expr: increase(prometheus_sd_refresh_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[10m]) > 0 + for: {{ dig "PrometheusSDRefreshFailure" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusSDRefreshFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotificationQueueRunningFull | default false) }} + - alert: PrometheusNotificationQueueRunningFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Alert notification queue of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is running full. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotificationqueuerunningfull + summary: Prometheus alert notification queue predicted to run full in less than 30m. + expr: |- + # Without min_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) + > + min_over_time(prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusNotificationQueueRunningFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotificationQueueRunningFull" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToSomeAlertmanagers | default false) }} + - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.alertmanager{{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstosomealertmanagers + summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager. + expr: |- + ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + * 100 + > 1 + for: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotConnectedToAlertmanagers | default false) }} + - alert: PrometheusNotConnectedToAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not connected to any Alertmanagers. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotconnectedtoalertmanagers + summary: Prometheus is not connected to any Alertmanagers. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) < 1 + for: {{ dig "PrometheusNotConnectedToAlertmanagers" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotConnectedToAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBReloadsFailing | default false) }} + - alert: PrometheusTSDBReloadsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} reload failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbreloadsfailing + summary: Prometheus has issues reloading blocks from disk. + expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBReloadsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBReloadsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBCompactionsFailing | default false) }} + - alert: PrometheusTSDBCompactionsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbcompactionsfailing + summary: Prometheus has issues compacting blocks. + expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBCompactionsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBCompactionsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotIngestingSamples | default false) }} + - alert: PrometheusNotIngestingSamples + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not ingesting samples. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotingestingsamples + summary: Prometheus is not ingesting samples. + expr: |- + ( + sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) <= 0 + and + ( + sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + or + sum without(rule_group) (prometheus_rule_group_rules{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + ) + ) + for: {{ dig "PrometheusNotIngestingSamples" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotIngestingSamples" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusDuplicateTimestamps | default false) }} + - alert: PrometheusDuplicateTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with different values but duplicated timestamp. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusduplicatetimestamps + summary: Prometheus is dropping samples with duplicate timestamps. + expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusDuplicateTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusDuplicateTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOutOfOrderTimestamps | default false) }} + - alert: PrometheusOutOfOrderTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with timestamps arriving out of order. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusoutofordertimestamps + summary: Prometheus drops samples with out-of-order timestamps. + expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOutOfOrderTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOutOfOrderTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteStorageFailures | default false) }} + - alert: PrometheusRemoteStorageFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotestoragefailures + summary: Prometheus fails to send samples to remote storage. + expr: |- + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + / + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + + + (rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + ) + ) + * 100 + > 1 + for: {{ dig "PrometheusRemoteStorageFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteStorageFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteBehind | default false) }} + - alert: PrometheusRemoteWriteBehind + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritebehind + summary: Prometheus remote write is behind. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + - ignoring(remote_name, url) group_right + max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 120 + for: {{ dig "PrometheusRemoteWriteBehind" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteBehind" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteDesiredShards | default false) }} + - alert: PrometheusRemoteWriteDesiredShards + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritedesiredshards + summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_shards_desired{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + > + max_over_time(prometheus_remote_storage_shards_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusRemoteWriteDesiredShards" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteDesiredShards" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRuleFailures | default false) }} + - alert: PrometheusRuleFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to evaluate {{`{{`}} printf "%.0f" $value {{`}}`}} rules in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusrulefailures + summary: Prometheus is failing rule evaluations. + expr: increase(prometheus_rule_evaluation_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusRuleFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRuleFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusMissingRuleEvaluations | default false) }} + - alert: PrometheusMissingRuleEvaluations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has missed {{`{{`}} printf "%.0f" $value {{`}}`}} rule group evaluations in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusmissingruleevaluations + summary: Prometheus is missing rule evaluations due to slow rule group evaluation. + expr: increase(prometheus_rule_group_iterations_missed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusMissingRuleEvaluations" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusMissingRuleEvaluations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetLimitHit | default false) }} + - alert: PrometheusTargetLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because the number of targets exceeded the configured target_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetlimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit. + expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusTargetLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusLabelLimitHit | default false) }} + - alert: PrometheusLabelLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuslabellimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit. + expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusLabelLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusLabelLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeBodySizeLimitHit | default false) }} + - alert: PrometheusScrapeBodySizeLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured body_size_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapebodysizelimithit + summary: Prometheus has dropped some targets that exceeded body size limit. + expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeBodySizeLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeBodySizeLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeSampleLimitHit | default false) }} + - alert: PrometheusScrapeSampleLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured sample_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapesamplelimithit + summary: Prometheus has failed scrapes that have exceeded the configured sample limit. + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeSampleLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeSampleLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetSyncFailure | default false) }} + - alert: PrometheusTargetSyncFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.0f" $value {{`}}`}} targets in Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} have failed to sync because invalid configuration was supplied.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetsyncfailure + summary: Prometheus has failed to sync targets. + expr: increase(prometheus_target_sync_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[30m]) > 0 + for: {{ dig "PrometheusTargetSyncFailure" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetSyncFailure" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusHighQueryLoad | default false) }} + - alert: PrometheusHighQueryLoad + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} query API has less than 20% available capacity in its query engine for the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheushighqueryload + summary: Prometheus is reaching its maximum capacity serving concurrent requests. + expr: avg_over_time(prometheus_engine_queries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.8 + for: {{ dig "PrometheusHighQueryLoad" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusHighQueryLoad" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToAnyAlertmanager | default false) }} + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstoanyalertmanager + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without (alertmanager) ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + ) + * 100 + > 3 + for: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml new file mode 100644 index 0000000000..7c25553861 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml @@ -0,0 +1,301 @@ +{{- /* +Generated from 'windows.node.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.node.rules + rules: + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + windows_system_system_up_time{job="windows-exporter"} + ) + record: node:windows_node:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, core) ( + windows_cpu_time_total{job="windows-exporter"} + )) + record: node:windows_node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m])) + record: :windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m]) + ) + record: node:windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"}) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: ':windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"} + windows_memory_cache_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemFreeCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: (windows_memory_cache_bytes{job="windows-exporter"} + windows_memory_modified_page_list_bytes{job="windows-exporter"} + windows_memory_standby_cache_core_bytes{job="windows-exporter"} + windows_memory_standby_cache_normal_priority_bytes{job="windows-exporter"} + windows_memory_standby_cache_reserve_bytes{job="windows-exporter"}) + record: node:windows_node_memory_totalCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemTotal_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (windows_memory_available_bytes{job="windows-exporter"}) + ) + record: node:windows_node_memory_bytes_available:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + windows_os_visible_memory_bytes{job="windows-exporter"} + ) + record: node:windows_node_memory_bytes_total:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + (node:windows_node_memory_bytes_total:sum - node:windows_node_memory_bytes_available:sum) + / + scalar(sum(node:windows_node_memory_bytes_total:sum)) + record: node:windows_node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - (node:windows_node_memory_bytes_available:sum / node:windows_node_memory_bytes_total:sum) + record: 'node:windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: irate(windows_memory_swap_page_operations_total{job="windows-exporter"}[5m]) + record: node:windows_node_memory_swap_io_pages:irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m]) + ) + record: :windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,instance,volume)( + (windows_logical_disk_size_bytes{job="windows-exporter"} + - windows_logical_disk_free_bytes{job="windows-exporter"}) + / windows_logical_disk_size_bytes{job="windows-exporter"} + ) + record: 'node:windows_node_filesystem_usage:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, volume) (windows_logical_disk_free_bytes{job="windows-exporter"} / windows_logical_disk_size_bytes{job="windows-exporter"}) + record: 'node:windows_node_filesystem_avail:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + record: :windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m])) + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + record: :windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m]) + + irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml new file mode 100644 index 0000000000..86340b5c05 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml @@ -0,0 +1,158 @@ +{{- /* +Generated from 'windows.pod.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.pod.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.pod.rules + rules: + - expr: windows_container_available{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_pod_container_available + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_cpu_usage_seconds_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_total_runtime + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_commit_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_memory_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_private_working_set_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_private_working_set_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_receive_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_received_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_transmit_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_transmitted_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + rate(windows_container_total_runtime{}[5m]) + ) + record: namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/secret.yaml new file mode 100644 index 0000000000..e4a1e73c7b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/secret.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.objectStorageConfig}} +{{- if and .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret (not .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + object-storage-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/service.yaml new file mode 100644 index 0000000000..d61b9d6ef3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/service.yaml @@ -0,0 +1,80 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + self-monitor: {{ .Values.prometheus.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.service.labels }} +{{ toYaml .Values.prometheus.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheus.service.clusterIP }} + clusterIP: {{ .Values.prometheus.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.service.nodePort }} + {{- end }} + port: {{ .Values.prometheus.service.port }} + targetPort: {{ .Values.prometheus.service.targetPort }} + - name: reloader-web + {{- if semverCompare "> 1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: {{ .Values.prometheus.service.reloaderWebPort }} + targetPort: reloader-web + {{- if .Values.prometheus.thanosIngress.enabled }} + - name: grpc + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosIngress.nodePort }} + {{- end }} + port: {{ .Values.prometheus.thanosIngress.servicePort }} + targetPort: {{ .Values.prometheus.thanosIngress.servicePort }} + {{- end }} +{{- if .Values.prometheus.service.additionalPorts }} +{{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} +{{- end }} + publishNotReadyAddresses: {{ .Values.prometheus.service.publishNotReadyAddresses }} + selector: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- if .Values.prometheus.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.prometheus.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.prometheus.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.prometheus.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceThanosSidecar.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceThanosSidecar.yaml new file mode 100644 index 0000000000..15b89c8c23 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceThanosSidecar.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-discovery + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosService.labels }} +{{ toYaml .Values.prometheus.thanosService.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosService.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosService.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosService.type }} + clusterIP: {{ .Values.prometheus.thanosService.clusterIP }} +{{- if ne .Values.prometheus.thanosService.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosService.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosService.portName }} + port: {{ .Values.prometheus.thanosService.port }} + targetPort: {{ .Values.prometheus.thanosService.targetPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosService.httpPortName }} + port: {{ .Values.prometheus.thanosService.httpPort }} + targetPort: {{ .Values.prometheus.thanosService.targetHttpPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml new file mode 100644 index 0000000000..453eed7f1b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosServiceExternal.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-external + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosServiceExternal.labels }} +{{ toYaml .Values.prometheus.thanosServiceExternal.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosServiceExternal.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosServiceExternal.type }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.thanosServiceExternal.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.thanosServiceExternal.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosServiceExternal.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosServiceExternal.portName }} + port: {{ .Values.prometheus.thanosServiceExternal.port }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosServiceExternal.httpPortName }} + port: {{ .Values.prometheus.thanosServiceExternal.httpPort }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetHttpPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceaccount.yaml new file mode 100644 index 0000000000..e97b989bbd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.prometheus.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.prometheus.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/servicemonitor.yaml new file mode 100644 index 0000000000..a36f3e33ca --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/servicemonitor.yaml @@ -0,0 +1,97 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.serviceMonitor.bearerTokenFile }} + {{- end }} + path: "{{ trimSuffix "/" .Values.prometheus.prometheusSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + - port: reloader-web + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "/metrics" + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.prometheus.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.prometheus.serviceMonitor.interval .interval }} + interval: {{ default $.Values.prometheus.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.prometheus.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.prometheus.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml new file mode 100644 index 0000000000..0f70aabb58 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.prometheus.thanosService.enabled .Values.prometheus.thanosServiceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-sidecar + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-sidecar +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.thanosServiceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.thanosServiceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.thanosService.httpPortName }} + {{- if .Values.prometheus.thanosServiceMonitor.interval }} + interval: {{ .Values.prometheus.thanosServiceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.scheme }} + scheme: {{ .Values.prometheus.thanosServiceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.prometheus.thanosServiceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + {{- end }} + path: "/metrics" + metricRelabelings: + {{- if .Values.prometheus.thanosServiceMonitor.metricRelabelings}} + {{ tpl (toYaml .Values.prometheus.thanosServiceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheus.thanosServiceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.thanosServiceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/servicemonitors.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/servicemonitors.yaml new file mode 100644 index 0000000000..a7a301babc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/servicemonitors.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalServiceMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalServiceMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + endpoints: +{{ toYaml .endpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .targetLabels }} + targetLabels: +{{ toYaml .targetLabels | indent 8 }} + {{- end }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .metricRelabelings }} + metricRelabelings: +{{ toYaml .metricRelabelings | indent 8 }} + {{- end }} + {{- if .relabelings }} + relabelings: +{{ toYaml .relabelings | indent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceperreplica.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceperreplica.yaml new file mode 100644 index 0000000000..4bc7f7b869 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/prometheus/serviceperreplica.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $serviceValues := .Values.prometheus.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.prometheus.prometheusSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + {{- if $.Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + statefulset.kubernetes.io/pod-name: prom-agent-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- else }} + app.kubernetes.io/name: prometheus + statefulset.kubernetes.io/pod-name: prometheus-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" $ }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/clusterrole.yaml new file mode 100644 index 0000000000..56ca9f5eae --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/clusterrole.yaml @@ -0,0 +1,135 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-admin + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-edit + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-ui-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" +{{- if .Values.grafana.enabled }} + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" +{{- end }} + verbs: + - 'get' + - 'create' +- apiGroups: + - "" + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- if .Values.grafana.enabled }} + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +{{- end }} + resources: + - endpoints + verbs: + - list +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/config-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/config-role.yaml new file mode 100644 index 0000000000..f48ffc827e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/config-role.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-admin + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-edit + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-view + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboard-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboard-role.yaml new file mode 100644 index 0000000000..d2f81976a2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboard-role.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create .Values.grafana.enabled }} +{{- if .Values.grafana.defaultDashboardsEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-admin + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-edit + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-view + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml new file mode 100644 index 0000000000..7b51a0bf7a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.ingressNginx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "ingress-nginx" | trunc 63 | trimSuffix "-" }} + {{- if .Values.grafana.sidecar.dashboards.annotations }} + annotations: {{ toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} + {{- end }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/ingress-nginx/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml new file mode 100644 index 0000000000..d73b257451 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-cluster + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/cluster/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml new file mode 100644 index 0000000000..8865efa932 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-home + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/home/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml new file mode 100644 index 0000000000..9b05cea2e8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-fleet-dashboards + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/fleet/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml new file mode 100644 index 0000000000..2afae10ef7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml @@ -0,0 +1,31 @@ +{{- $files := (.Files.Glob "files/rancher/k8s/*").AsConfig }} +{{- $filesDict := (fromYaml $files) }} +{{- if not (include "exporter.kubeEtcd.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-etcd-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-etcd.json") -}} +{{- end }} +{{- if not (include "exporter.kubeControllerManager.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-k8s-components-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-k8s-components.json") -}} +{{- else }} +{{- $_ := (set $filesDict "rancher-k8s-components-nodes.json" (get $filesDict "rancher-k8s-components-nodes.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- $_ := (set $filesDict "rancher-k8s-components.json" (get $filesDict "rancher-k8s-components.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- end }} +{{ $files = (toYaml $filesDict) }} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-k8s + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ $files | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml new file mode 100644 index 0000000000..172c36e9d1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-nodes + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/nodes/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml new file mode 100644 index 0000000000..19836ec4e4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml @@ -0,0 +1,18 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.rancherMonitoring.enabled $selector }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-performance-debugging + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/performance/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml new file mode 100644 index 0000000000..940f18869b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-pods + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/pods/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml new file mode 100644 index 0000000000..d146dacdd0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-workloads + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/workloads/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml new file mode 100644 index 0000000000..90d24c2061 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml @@ -0,0 +1,53 @@ +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-fleet-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: fleet + selector: + matchLabels: + app: fleet-controller +{{- end }} +--- +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-gitops-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: gitops + selector: + matchLabels: + app: gitjob +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml new file mode 100644 index 0000000000..53a9ad6897 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml @@ -0,0 +1,27 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + jobLabel: ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: {{ .Values.ingressNginx.namespace }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.ingressNginx.service.port }} + protocol: TCP + targetPort: {{ .Values.ingressNginx.service.targetPort }} + selector: + {{- if .Values.ingressNginx.service.selector }} +{{ toYaml .Values.ingressNginx.service.selector | indent 4 }} + {{- else }} + app: ingress-nginx + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml new file mode 100644 index 0000000000..b0f92e63b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml @@ -0,0 +1,49 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + namespace: {{ .Values.ingressNginx.namespace }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Values.ingressNginx.namespace }} + endpoints: + - port: http-metrics + {{- if .Values.ingressNginx.serviceMonitor.interval}} + interval: {{ .Values.ingressNginx.serviceMonitor.interval }} + {{- end }} + {{- if .Values.ingressNginx.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.ingressNginx.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.ingressNginx.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.ingressNginx.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.ingressNginx.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.ingressNginx.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml new file mode 100644 index 0000000000..1fba8f23f7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.rancherMonitoring.enabled $selector }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: rancher + namespace: cattle-system +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + port: http + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + serverName: rancher + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: rancher +{{- if .Values.rancherMonitoring.namespaceSelector }} + namespaceSelector: {{ .Values.rancherMonitoring.namespaceSelector | toYaml | nindent 4 }} +{{- end }} + selector: {{ include "rancher.serviceMonitor.selector" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +rules: +- apiGroups: + - management.cattle.io + resources: + - ranchermetrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/hardened.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/hardened.yaml new file mode 100644 index 0000000000..f9a66151ee --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/hardened.yaml @@ -0,0 +1,147 @@ +{{- $namespaces := dict "_0" .Release.Namespace -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) -}} +{{- $_ := set $namespaces "_1" .Values.grafana.defaultDashboards.namespace -}} +{{- end -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa + spec: + serviceAccountName: {{ .Chart.Name }}-patch-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + {{- range $_, $ns := $namespaces }} + - name: patch-sa-{{ $ns }} + image: {{ template "system_default_registry" $ }}{{ $.Values.global.kubectl.repository }}:{{ $.Values.global.kubectl.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", "{{ $ns }}"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +rules: +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: ['get', 'patch'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-patch-sa +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-patch-sa +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- range $_, $ns := $namespaces }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ $ns }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +{{- end }} +{{- end }} +--- +{{- if .Values.hardened.k3s.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rancher-monitoring-coredns-allow-all + namespace: kube-system +spec: + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + podSelector: + matchLabels: + k8s-app: kube-dns +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml new file mode 100644 index 0000000000..53cb898214 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "0" +data: +{{ (.Files.Glob "files/upgrade/scripts/*").AsConfig | indent 2 }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/upgrade/job.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/upgrade/job.yaml new file mode 100644 index 0000000000..8f2771740c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/upgrade/job.yaml @@ -0,0 +1,46 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "2" +spec: + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + spec: + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + securityContext: + runAsNonRoot: false + runAsUser: 0 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + - name: run-scripts + image: {{ template "system_default_registry" . }}{{ .Values.upgrade.image.repository }}:{{ .Values.upgrade.image.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: + - /bin/sh + - -c + - > + for s in $(find /etc/scripts -type f); do + echo "Running $s..."; + cat $s | bash + done; + volumeMounts: + - name: upgrade + mountPath: /etc/scripts + volumes: + - name: upgrade + configMap: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml new file mode 100644 index 0000000000..e929a19925 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml @@ -0,0 +1,131 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded + "helm.sh/hook-weight": "1" +rules: +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - 'list' + - 'delete' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/extrasecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/extrasecret.yaml new file mode 100644 index 0000000000..587fca2dca --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.thanosRuler.extraSecret.data -}} +{{- $secretName := printf "%s-extra" (include "kube-prometheus-stack.thanosRuler.name" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.thanosRuler.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.extraSecret.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.thanosRuler.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/ingress.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/ingress.yaml new file mode 100644 index 0000000000..e245ad448e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.ingress.enabled }} +{{- $pathType := .Values.thanosRuler.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := include "kube-prometheus-stack.thanosRuler.name" . }} +{{- $servicePort := .Values.thanosRuler.service.port -}} +{{- $routePrefix := list .Values.thanosRuler.thanosRulerSpec.routePrefix }} +{{- $paths := .Values.thanosRuler.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.thanosRuler.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- if .Values.thanosRuler.ingress.labels }} +{{ toYaml .Values.thanosRuler.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.thanosRuler.ingress.ingressClassName }} + ingressClassName: {{ .Values.thanosRuler.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.thanosRuler.ingress.hosts }} + {{- range $host := .Values.thanosRuler.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.thanosRuler.ingress.tls }} + tls: +{{ tpl (toYaml .Values.thanosRuler.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml new file mode 100644 index 0000000000..83e54edf9b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.thanosRuler.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.thanosRuler.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/ruler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/ruler.yaml new file mode 100644 index 0000000000..b281221563 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/ruler.yaml @@ -0,0 +1,189 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ThanosRuler +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.thanosRulerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.thanosRuler.thanosRulerSpec.image.registry -}} + {{- if and .Values.thanosRuler.thanosRulerSpec.image.tag .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}" + {{- end }} + {{- if .Values.thanosRuler.thanosRulerSpec.image.sha }} + sha: {{ .Values.thanosRuler.thanosRulerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.thanosRuler.thanosRulerSpec.replicas }} + listenLocal: {{ .Values.thanosRuler.thanosRulerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} +{{- if .Values.thanosRuler.thanosRulerSpec.externalPrefix }} + externalPrefix: "{{ tpl .Values.thanosRuler.thanosRulerSpec.externalPrefix . }}" +{{- else if and .Values.thanosRuler.ingress.enabled .Values.thanosRuler.ingress.hosts }} + externalPrefix: "http://{{ tpl (index .Values.thanosRuler.ingress.hosts 0) . }}{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- else }} + externalPrefix: http://{{ template "kube-prometheus-stack.thanosRuler.name" . }}.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.thanosRuler.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.additionalArgs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.nodeSelector }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.thanosRuler.thanosRulerSpec.paused }} + logFormat: {{ .Values.thanosRuler.thanosRulerSpec.logFormat | quote }} + logLevel: {{ .Values.thanosRuler.thanosRulerSpec.logLevel | quote }} + retention: {{ .Values.thanosRuler.thanosRulerSpec.retention | quote }} +{{- if .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} + evaluationInterval: {{ .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleSelector | indent 4) .}} +{{- else if .Values.thanosRuler.thanosRulerSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }} + alertQueryUrl: "{{ .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }}" +{{- end}} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl }} + alertmanagersUrl: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret }} + alertmanagersConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.secret }} + alertmanagersConfig: + key: alertmanager-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryEndpoints }} + queryEndpoints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.queryEndpoints | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret }} + queryConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.queryConfig.secret }} + queryConfig: + key: query-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.resources }} + resources: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.routePrefix }} + routePrefix: "{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.securityContext }} + securityContext: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.storage }} + storage: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.storage | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret }} + objectStorageConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.secret }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.labels }} + labels: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.thanosRuler.thanosRulerSpec.podAntiAffinity .Values.thanosRuler.thanosRulerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.affinity }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- else if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.tolerations }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.containers }} + containers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.initContainers }} + initContainers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.priorityClassName }} + priorityClassName: {{.Values.thanosRuler.thanosRulerSpec.priorityClassName }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumes }} + volumes: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.thanosRuler.thanosRulerSpec.portName }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/secret.yaml new file mode 100644 index 0000000000..acab7fd9ae --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/secret.yaml @@ -0,0 +1,26 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + {{- with .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig }} + {{- if and .secret (not .existingSecret) }} + alertmanager-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.objectStorageConfig }} + {{- if and .secret (not .existingSecret) }} + object-storage-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.queryConfig }} + {{- if and .secret (not .existingSecret) }} + query-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/service.yaml new file mode 100644 index 0000000000..be0c844591 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/service.yaml @@ -0,0 +1,53 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.service.labels }} +{{ toYaml .Values.thanosRuler.service.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.service.clusterIP }} + clusterIP: {{ .Values.thanosRuler.service.clusterIP }} +{{- end }} +{{- if .Values.thanosRuler.service.externalIPs }} + externalIPs: +{{ toYaml .Values.thanosRuler.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.thanosRuler.service.loadBalancerIP }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.thanosRuler.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.thanosRuler.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.thanosRuler.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if eq .Values.thanosRuler.service.type "NodePort" }} + nodePort: {{ .Values.thanosRuler.service.nodePort }} + {{- end }} + port: {{ .Values.thanosRuler.service.port }} + targetPort: {{ .Values.thanosRuler.service.targetPort }} + protocol: TCP +{{- if .Values.thanosRuler.service.additionalPorts }} +{{ toYaml .Values.thanosRuler.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + type: "{{ .Values.thanosRuler.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/serviceaccount.yaml new file mode 100644 index 0000000000..b58f1cd4df --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/servicemonitor.yaml new file mode 100644 index 0000000000..b2b138b498 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/thanos-ruler/servicemonitor.yaml @@ -0,0 +1,82 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.thanosRuler.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.thanosRuler.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + release: {{ $.Release.Name | quote }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if .Values.thanosRuler.serviceMonitor.interval }} + interval: {{ .Values.thanosRuler.serviceMonitor.interval }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.thanosRuler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.scheme }} + scheme: {{ .Values.thanosRuler.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.thanosRuler.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.thanosRuler.thanosRulerSpec.routePrefix }}/metrics" + {{- if .Values.thanosRuler.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.thanosRuler.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.thanosRuler.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.thanosRuler.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.thanosRuler.serviceMonitor.interval .interval }} + interval: {{ default $.Values.thanosRuler.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.thanosRuler.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..6fcb8b3a69 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/validate-install-crd.yaml @@ -0,0 +1,23 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/AlertmanagerConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/Alertmanager" false -}} +# {{- set $found "monitoring.coreos.com/v1/PodMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/Probe" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/PrometheusAgent" false -}} +# {{- set $found "monitoring.coreos.com/v1/Prometheus" false -}} +# {{- set $found "monitoring.coreos.com/v1/PrometheusRule" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/ScrapeConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/ThanosRuler" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/values.yaml new file mode 100644 index 0000000000..0bf19a8572 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3.up57.0.3/values.yaml @@ -0,0 +1,5431 @@ +# Default values for kube-prometheus-stack. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Rancher Monitoring Configuration + +## Configuration for prometheus-adapter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter +## +prometheus-adapter: + enabled: true + prometheus: + # Change this if you change the namespaceOverride or nameOverride of prometheus-operator + url: http://rancher-monitoring-prometheus.cattle-monitoring-system.svc + port: 9090 + +## RKE PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rkeControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.23" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.23 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeEtcd: + enabled: false + metricsPort: 2379 + component: kube-etcd + clients: + port: 10014 + https: + enabled: true + certDir: /etc/kubernetes/ssl + certFile: kube-etcd-*.pem + keyFile: kube-etcd-*-key.pem + caCertFile: kube-ca.pem + seLinuxOptions: + # Gives rkeEtcd permissions to read files in /etc/kubernetes/* + # Type is defined in https://github.com/rancher/rancher-selinux + type: rke_kubereader_t + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeIngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + clients: + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + nodeSelector: + node-role.kubernetes.io/worker: "true" + +## k3s PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +k3sServer: + enabled: false + metricsPort: 10250 + component: k3s-server + clients: + port: 10013 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +hardened: + k3s: + networkPolicy: + enabled: true + +## KubeADM PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +kubeAdmControllerManager: + enabled: false + metricsPort: 10257 + component: kube-controller-manager + clients: + port: 10011 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + port: 10012 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmEtcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## rke2 PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rke2ControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Scheduler: + enabled: false + metricsPort: 10259 # default to secure port as of k8s >= 1.22 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Proxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2Etcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2IngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + # in the RKE2 cluster, the ingress-nginx-controller is deployed + # as a non-hostNetwork workload starting at the following versions + # - >= v1.22.12+rke2r1 < 1.23.0-0 + # - >= v1.23.9+rke2r1 < 1.24.0-0 + # - >= v1.24.3+rke2r1 < 1.25.0-0 + # - >= v1.25.0+rke2r1 + # As a result we do not need clients and proxies as we can directly create + # a service that targets the workload with the given app name + namespaceOverride: kube-system + clients: + enabled: false + proxy: + enabled: false + service: + selector: + app.kubernetes.io/name: rke2-ingress-nginx + kubeVersionOverrides: + - constraint: "< 1.21.0-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a DaemonSet with 1 pod when RKE2 version is < 1.21.0-0 + deployment: + enabled: false + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.21.0-0 < 1.22.12-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.21.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.23.0-0 < v1.23.9-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.24.0-0 < v1.24.3-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + + + +## Additional PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## + +# hardenedKubelet can only be deployed if kubelet.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default kubelet option with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedKubelet: + enabled: false + metricsPort: 10250 + component: kubelet + clients: + port: 10015 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +# hardenedNodeExporter can only be deployed if nodeExporter.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default nodeExporter with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedNodeExporter: + enabled: false + metricsPort: 9796 + component: node-exporter + clients: + port: 10016 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## Upgrades +upgrade: + ## Run upgrade scripts before an upgrade or rollback via a Job hook + enabled: true + ## Image to use to run the scripts + image: + repository: rancher/shell + tag: v0.2.1 + +## Rancher Monitoring +## + +rancherMonitoring: + enabled: true + + ## A namespaceSelector to identify the namespace to find the Rancher deployment + ## + namespaceSelector: + matchNames: + - cattle-system + + ## A selector to identify the Rancher deployment + ## If not set, the chart will try to search for the Rancher deployment in the cattle-system namespace and infer the selector values from it + ## If the Rancher deployment does not exist, no resources will be deployed. + ## + selector: {} + +## Component scraping nginx-ingress-controller +## +ingressNginx: + enabled: false + + ## The namespace to search for your nginx-ingress-controller + ## + namespace: ingress-nginx + + service: + port: 9913 + targetPort: 10254 + # selector: + # app: ingress-nginx + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "30s" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +# Prometheus Operator Configuration + +## Provide a name in place of kube-prometheus-stack for `app:` labels +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +nameOverride: "rancher-monitoring" + +## Override the deployment namespace +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +namespaceOverride: "cattle-monitoring-system" + +## Provide a k8s version to auto dashboard import script example: kubeTargetVersionOverride: 1.26.6 +## +kubeTargetVersionOverride: "" + +## Allow kubeVersion to be overridden while creating the ingress +## +kubeVersionOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +## Install Prometheus Operator CRDs +## +crds: + enabled: true + +## custom Rules to override "for" and "severity" in defaultRules +## +customRules: {} + # AlertmanagerFailedReload: + # for: 3m + # AlertmanagerMembersInconsistent: + # for: 5m + # severity: "warning" + +## Create default rules for monitoring the cluster +## +defaultRules: + create: true + rules: + alertmanager: true + etcd: true + configReloaders: true + general: true + k8sContainerCpuUsageSecondsTotal: true + k8sContainerMemoryCache: true + k8sContainerMemoryRss: true + k8sContainerMemorySwap: true + k8sContainerResource: true + k8sContainerMemoryWorkingSetBytes: true + k8sPodOwner: true + kubeApiserverAvailability: true + kubeApiserverBurnrate: true + kubeApiserverHistogram: true + kubeApiserverSlos: true + kubeControllerManager: true + kubelet: true + kubeProxy: true + kubePrometheusGeneral: true + kubePrometheusNodeRecording: true + kubernetesApps: true + kubernetesResources: true + kubernetesStorage: true + kubernetesSystem: true + kubeSchedulerAlerting: true + kubeSchedulerRecording: true + kubeStateMetrics: true + network: true + node: true + nodeExporterAlerting: true + nodeExporterRecording: true + prometheus: true + prometheusOperator: true + windows: true + + ## Reduce app namespace alert scope + appNamespacesTarget: ".*" + + ## Set keep_firing_for for all alerts + keepFiringFor: "" + + ## Labels for default rules + labels: {} + ## Annotations for default rules + annotations: {} + + ## Additional labels for PrometheusRule alerts + additionalRuleLabels: {} + + ## Additional annotations for PrometheusRule alerts + additionalRuleAnnotations: {} + + ## Additional labels for specific PrometheusRule alert groups + additionalRuleGroupLabels: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + ## Additional annotations for specific PrometheusRule alerts groups + additionalRuleGroupAnnotations: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + additionalAggregationLabels: [] + + ## Prefix for runbook URLs. Use this to override the first part of the runbookURLs that is common to all rules. + runbookUrl: "https://runbooks.prometheus-operator.dev/runbooks" + + ## Disabled PrometheusRule alerts + disabled: {} + # KubeAPIDown: true + # NodeRAIDDegraded: true + +## Deprecated way to provide custom recording or alerting rules to be deployed into the cluster. +## +# additionalPrometheusRules: [] +# - name: my-rule-file +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## Provide custom recording or alerting rules to be deployed into the cluster. +## +additionalPrometheusRulesMap: {} +# rule-name: +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## +global: + cattle: + psp: + enabled: false + + systemDefaultRegistry: "" + ## Windows Monitoring + ## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-windows-exporter + ## + ## Deploys a DaemonSet of Prometheus exporters based on https://github.com/prometheus-community/windows_exporter. + ## Every Windows host must have a wins version of 0.1.0+ to use this chart (default as of Rancher 2.5.8). + ## To upgrade wins versions on Windows hosts, see https://github.com/rancher/wins/tree/master/charts/rancher-wins-upgrader. + ## + windows: + enabled: false + seLinux: + enabled: false + kubectl: + repository: rancher/kubectl + tag: v1.20.2 + pullPolicy: IfNotPresent + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Global image registry to use if it needs to be overriden for some specific use cases (e.g local registries, custom images, ...) + ## + imageRegistry: docker.io + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + # or + # - "image-pull-secret" + +windowsMonitoring: + ## Deploys the windows-exporter and Windows-specific dashboards and rules (job name must be 'windows-exporter') + enabled: false + +## Configuration for prometheus-windows-exporter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-windows-exporter +## +prometheus-windows-exporter: + ## Enable ServiceMonitor and set Kubernetes label to use as a job label + ## + prometheus: + monitor: + enabled: true + jobLabel: jobLabel + + releaseLabel: true + + ## Set job label to 'windows-exporter' as required by the default Prometheus rules and Grafana dashboards + ## + podLabels: + jobLabel: windows-exporter + + ## Enable memory and container metrics as required by the default Prometheus rules and Grafana dashboards + ## + config: |- + collectors: + enabled: '[defaults],memory,container' + +## Configuration for alertmanager +## ref: https://prometheus.io/docs/alerting/alertmanager/ +## +alertmanager: + + ## Deploy alertmanager + ## + enabled: true + + ## Annotations for Alertmanager + ## + annotations: {} + + ## Api that prometheus will use to communicate with alertmanager. Possible values are v1, v2 + ## + apiVersion: v2 + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + ## Configure pod disruption budgets for Alertmanager + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + ## Alertmanager configuration directives + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + config: + global: + resolve_timeout: 5m + inhibit_rules: + - source_matchers: + - 'severity = critical' + target_matchers: + - 'severity =~ warning|info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'severity = warning' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'alertname = InfoInhibitor' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - target_matchers: + - 'alertname = InfoInhibitor' + route: + group_by: ['namespace'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'null' + routes: + - receiver: 'null' + matchers: + - alertname = "Watchdog" + receivers: + - name: 'null' + templates: + - '/etc/alertmanager/config/*.tmpl' + + ## Alertmanager configuration directives (as string type, preferred over the config hash map) + ## stringConfig will be used only, if tplConfig is true + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + stringConfig: "" + + ## Pass the Alertmanager configuration directives through Helm's templating + ## engine. If the Alertmanager configuration contains Alertmanager templates, + ## they'll need to be properly escaped so that they are not interpreted by + ## Helm + ## ref: https://helm.sh/docs/developing_charts/#using-the-tpl-function + ## https://prometheus.io/docs/alerting/configuration/#tmpl_string + ## https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + tplConfig: false + + ## Alertmanager template files to format alerts + ## By default, templateFiles are placed in /etc/alertmanager/config/ and if + ## they have a .tmpl file suffix will be loaded. See config.templates above + ## to change, add other suffixes. If adding other suffixes, be sure to update + ## config.templates above to include those suffixes. + ## ref: https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + ## + + templateFiles: + rancher_defaults.tmpl: |- + {{- define "slack.rancher.text" -}} + {{ template "rancher.text_multiple" . }} + {{- end -}} + + {{- define "rancher.text_multiple" -}} + *[GROUP - Details]* + One or more alarms in this group have triggered a notification. + + {{- if gt (len .GroupLabels.Values) 0 }} + *Group Labels:* + {{- range .GroupLabels.SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- if .ExternalURL }} + *Link to AlertManager:* {{ .ExternalURL }} + {{- end }} + + {{- range .Alerts }} + {{ template "rancher.text_single" . }} + {{- end }} + {{- end -}} + + {{- define "rancher.text_single" -}} + {{- if .Labels.alertname }} + *[ALERT - {{ .Labels.alertname }}]* + {{- else }} + *[ALERT]* + {{- end }} + {{- if .Labels.severity }} + *Severity:* `{{ .Labels.severity }}` + {{- end }} + {{- if .Labels.cluster }} + *Cluster:* {{ .Labels.cluster }} + {{- end }} + {{- if .Annotations.summary }} + *Summary:* {{ .Annotations.summary }} + {{- end }} + {{- if .Annotations.message }} + *Message:* {{ .Annotations.message }} + {{- end }} + {{- if .Annotations.description }} + *Description:* {{ .Annotations.description }} + {{- end }} + {{- if .Annotations.runbook_url }} + *Runbook URL:* <{{ .Annotations.runbook_url }}|:spiral_note_pad:> + {{- end }} + {{- with .Labels }} + {{- with .Remove (stringSlice "alertname" "severity" "cluster") }} + {{- if gt (len .) 0 }} + *Additional Labels:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Annotations }} + {{- with .Remove (stringSlice "summary" "message" "description" "runbook_url") }} + {{- if gt (len .) 0 }} + *Additional Annotations:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end -}} + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + + labels: {} + + ## Override ingress to a different defined port on the service + # servicePort: 8081 + ## Override ingress to a different service then the default, this is useful if you need to + ## point to a specific instance of the alertmanager (eg kube-prometheus-stack-alertmanager-0) + # serviceName: kube-prometheus-stack-alertmanager-0 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - alertmanager.domain.com + + ## Paths to use for ingress rules - one path should match the alertmanagerSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Alertmanager Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: alertmanager-general-tls + # hosts: + # - alertmanager.example.com + + ## Configuration for Alertmanager secret + ## + secret: + annotations: {} + + # by default the alertmanager secret is not overwritten if it already exists + recreateIfExists: false + + ## Configuration for creating an Ingress that will map to each Alertmanager replica service + ## alertmanager.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for alertmanager per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "alertmanager" + + ## Configuration for Alertmanager service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Alertmanager Service to listen on + ## + port: 9093 + ## To be used with a proxy extraContainer port + ## + targetPort: 9093 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30903 + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + + ## Additional ports to open for Alertmanager service + ## + additionalPorts: [] + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a separate Service for each statefulset Alertmanager replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Alertmanager Service per replica to listen on + ## + port: 9093 + + ## To be used with a proxy extraContainer port + targetPort: 9093 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30904 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a ServiceMonitor for AlertManager + ## + serviceMonitor: + ## If true, a ServiceMonitor will be created for the AlertManager service. + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## enableHttp2: Whether to enable HTTP2. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint + enableHttp2: true + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting alertmanagerSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec + ## + alertmanagerSpec: + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the Alertmanager pods. + ## + podMetadata: {} + + ## Image of Alertmanager + ## + image: + repository: rancher/mirrored-prometheus-alertmanager + tag: v0.27.0 + sha: "" + + ## If true then the user will be responsible to provide a secret with alertmanager configuration + ## So when true the config part will be ignored (including templateFiles) and the one in the secret will be used + ## + useExistingSecret: false + + ## Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the + ## Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/. + ## + secrets: [] + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. + ## The ConfigMaps are mounted into /etc/alertmanager/configmaps/. + ## + configMaps: [] + + ## ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for + ## this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config. + ## + # configSecret: + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerwebspec + web: {} + + ## AlertmanagerConfigs to be selected to merge and configure Alertmanager with. + ## + alertmanagerConfigSelector: {} + ## Example which selects all alertmanagerConfig resources + ## with label "alertconfig" with values any of "example-config" or "example-config-2" + # alertmanagerConfigSelector: + # matchExpressions: + # - key: alertconfig + # operator: In + # values: + # - example-config + # - example-config-2 + # + ## Example which selects all alertmanagerConfig resources with label "role" set to "example-config" + # alertmanagerConfigSelector: + # matchLabels: + # role: example-config + + ## Namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace. + ## + alertmanagerConfigNamespaceSelector: {} + ## Example which selects all namespaces + ## with label "alertmanagerconfig" with values any of "example-namespace" or "example-namespace-2" + # alertmanagerConfigNamespaceSelector: + # matchExpressions: + # - key: alertmanagerconfig + # operator: In + # values: + # - example-namespace + # - example-namespace-2 + + ## Example which selects all namespaces with label "alertmanagerconfig" set to "enabled" + # alertmanagerConfigNamespaceSelector: + # matchLabels: + # alertmanagerconfig: enabled + + ## AlermanagerConfig to be used as top level configuration + ## + alertmanagerConfiguration: {} + ## Example with select a global alertmanagerconfig + # alertmanagerConfiguration: + # name: global-alertmanager-Configuration + + ## Defines the strategy used by AlertmanagerConfig objects to match alerts. eg: + ## + alertmanagerConfigMatcherStrategy: {} + ## Example with use OnNamespace strategy + # alertmanagerConfigMatcherStrategy: + # type: OnNamespace + + ## Define Log Format + # Use logfmt (default) or json logging + logFormat: logfmt + + ## Log level for Alertmanager to be configured with. + ## + logLevel: info + + ## Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the + ## running cluster equal to the expected size. + replicas: 1 + + ## Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression + ## [0-9]+(ms|s|m|h) (milliseconds seconds minutes hours). + ## + retention: 120h + + ## Storage is the definition of how storage will be used by the Alertmanager instances. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storage: {} + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + + ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false + ## + externalUrl: + + ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, + ## but the server serves requests under a different route prefix. For example for use with kubectl proxy. + ## + routePrefix: / + + ## scheme: HTTP scheme to use. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when connect to the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + ## If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. + ## + paused: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Define resources requests and limits for single Pods. + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: + memory: 500Mi + cpu: 1000m + requests: + memory: 100Mi + cpu: 100m + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + ## + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the alertmanager instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## If specified, the pod's tolerations. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: alertmanager + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. + ## Note this is only for the Alertmanager UI, not the gossip communication. + ## + listenLocal: false + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. + ## + containers: [] + # containers: + # - name: oauth-proxy + # image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1 + # args: + # - --upstream=http://127.0.0.1:9093 + # - --http-address=0.0.0.0:8081 + # - --metrics-address=0.0.0.0:8082 + # - ... + # ports: + # - containerPort: 8081 + # name: oauth-proxy + # protocol: TCP + # - containerPort: 8082 + # name: oauth-metrics + # protocol: TCP + # resources: {} + + # Additional volumes on the output StatefulSet definition. + volumes: [] + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. + ## + additionalPeers: [] + + ## PortName to use for Alert Manager. + ## + portName: "http-web" + + ## ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 + ## + clusterAdvertiseAddress: false + + ## clusterGossipInterval determines interval between gossip attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterGossipInterval: "" + + ## clusterPeerTimeout determines timeout for cluster peering. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPeerTimeout: "" + + ## clusterPushpullInterval determines interval between pushpull attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPushpullInterval: "" + + ## ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. + ## Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. + forceEnableClusterMode: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use alertmanager.alertmanagerSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + +## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +## +grafana: + enabled: true + namespaceOverride: "" + + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + grafana.ini: + users: + auto_assign_org_role: Viewer + auth: + disable_login_form: false + auth.anonymous: + enabled: true + org_role: Viewer + auth.basic: + enabled: false + dashboards: + # Modify this value to change the default dashboard shown on the main Grafana page + default_home_dashboard_path: /tmp/dashboards/rancher-default-home.json + security: + # Required to embed dashboards in Rancher Cluster Overview Dashboard on Cluster Explorer + allow_embedding: true + + deploymentStrategy: + type: Recreate + + ## ForceDeployDatasources Create datasource configmap even if grafana deployment has been disabled + ## + forceDeployDatasources: false + + ## ForceDeployDashboard Create dashboard configmap even if grafana deployment has been disabled + ## + forceDeployDashboards: false + + ## Deploy default dashboards + ## + defaultDashboardsEnabled: true + + # Additional options for defaultDashboards + defaultDashboards: + # The default namespace to place defaultDashboards within + namespace: cattle-dashboards + # Whether to create the default namespace as a Helm managed namespace or use an existing namespace + # If false, the defaultDashboards.namespace will be created as a Helm managed namespace + useExistingNamespace: false + # Whether the Helm managed namespace created by this chart should be left behind on a Helm uninstall + # If you place other dashboards in this namespace, then they will be deleted on a helm uninstall + # Ignore if useExistingNamespace is true + cleanupOnUninstall: false + + ## Timezone for the default dashboards + ## Other options are: browser or a specific timezone, i.e. Europe/Luxembourg + ## + defaultDashboardsTimezone: utc + + ## Editable flag for the default dashboards + ## + defaultDashboardsEditable: true + + adminPassword: prom-operator + + ingress: + ## If true, Grafana Ingress will be created + ## + enabled: false + + ## IngressClassName for Grafana Ingress. + ## Should be provided if Ingress is enable. + ## + # ingressClassName: nginx + + ## Annotations for Grafana Ingress + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + + ## Labels to be added to the Ingress + ## + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enable. + ## + # hosts: + # - grafana.domain.com + hosts: [] + + ## Path for grafana ingress + path: / + + ## TLS configuration for grafana Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: grafana-general-tls + # hosts: + # - grafana.example.com + + # # To make Grafana persistent (Using Statefulset) + # # + # persistence: + # enabled: true + # type: sts + # storageClassName: "storageClassName" + # accessModes: + # - ReadWriteOnce + # size: 20Gi + # finalizers: + # - kubernetes.io/pvc-protection + + serviceAccount: + create: true + autoMount: true + + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + searchNamespace: cattle-dashboards + labelValue: "1" + + # Support for new table panels, when enabled grafana auto migrates the old table panels to newer table panels + enableNewTablePanelSyntax: false + + ## Annotations for Grafana dashboard configmaps + ## + annotations: {} + multicluster: + global: + enabled: false + etcd: + enabled: false + provider: + allowUiUpdates: false + datasources: + enabled: true + defaultDatasourceEnabled: true + isDefaultDatasource: true + + uid: prometheus + + ## URL of prometheus datasource + ## + # url: http://prometheus-stack-prometheus:9090/ + + ## Prometheus request timeout in seconds + # timeout: 30 + + # If not defined, will use prometheus.prometheusSpec.scrapeInterval or its default + # defaultDatasourceScrapeInterval: 15s + + ## Annotations for Grafana datasource configmaps + ## + annotations: {} + + ## Set method for HTTP to send query to datasource + httpMethod: POST + + ## Create datasource for each Pod of Prometheus StatefulSet; + ## this uses headless service `prometheus-operated` which is + ## created by Prometheus Operator + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/0fee93e12dc7c2ea1218f19ae25ec6b893460590/pkg/prometheus/statefulset.go#L255-L286 + createPrometheusReplicasDatasources: false + label: grafana_datasource + labelValue: "1" + + ## Field with internal link pointing to existing data source in Grafana. + ## Can be provisioned via additionalDataSources + exemplarTraceIdDestinations: {} + # datasourceUid: Jaeger + # traceIdLabelName: trace_id + alertmanager: + enabled: true + uid: alertmanager + handleGrafanaManagedAlerts: false + implementation: prometheus + + extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # configMap: certs-configmap + # readOnly: true + + deleteDatasources: [] + # - name: example-datasource + # orgId: 1 + + ## Configure additional grafana datasources (passed through tpl) + ## ref: http://docs.grafana.org/administration/provisioning/#datasources + additionalDataSources: [] + # - name: prometheus-sample + # access: proxy + # basicAuth: true + # basicAuthPassword: pass + # basicAuthUser: daco + # editable: false + # jsonData: + # tlsSkipVerify: true + # orgId: 1 + # type: prometheus + # url: https://{{ printf "%s-prometheus.svc" .Release.Name }}:9090 + # version: 1 + + ## Passed to grafana subchart and used by servicemonitor below + ## + service: + portName: nginx-http + ## Port for Grafana Service to listen on + ## + port: 80 + ## To be used with a proxy extraContainer port + ## + targetPort: 8080 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30950 + ## Service type + ## + type: ClusterIP + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod + extraContainers: | + - name: grafana-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }}" + ports: + - containerPort: 8080 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: grafana-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## Volumes that can be used in containers + extraContainerVolumes: + - name: nginx-home + emptyDir: {} + - name: grafana-nginx + configMap: + name: grafana-nginx-proxy-config + items: + - key: nginx.conf + mode: 438 + path: nginx.conf + + ## If true, create a serviceMonitor for grafana + ## + serviceMonitor: + # If true, a ServiceMonitor CRD is created for a prometheus operator + # https://github.com/coreos/prometheus-operator + # + enabled: true + + # Path to use for scraping metrics. Might be different if server.root_url is set + # in grafana.ini + path: "/metrics" + + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + + # labels for the ServiceMonitor + labels: {} + + # Scrape interval. If not set, the Prometheus default scrape interval is used. + # + interval: "" + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + resources: + limits: + memory: 200Mi + cpu: 200m + requests: + memory: 100Mi + cpu: 100m + + testFramework: + enabled: false + +## Flag to disable all the kubernetes component scrapers +## +kubernetesServiceMonitors: + enabled: true + +## Component scraping the kube api server +## +kubeApiServer: + enabled: true + tlsConfig: + serverName: kubernetes + insecureSkipVerify: false + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: component + selector: + matchLabels: + component: apiserver + provider: kubernetes + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: + # Drop excessively noisy apiserver buckets. + - action: drop + regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50) + sourceLabels: + - __name__ + - le + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: + # - __meta_kubernetes_namespace + # - __meta_kubernetes_service_name + # - __meta_kubernetes_endpoint_port_name + # action: keep + # regex: default;kubernetes;https + # - targetLabel: __address__ + # replacement: kubernetes.default.svc:443 + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kubelet and kubelet-hosted cAdvisor +## +kubelet: + enabled: true + namespace: kube-system + + serviceMonitor: + ## Attach metadata to discovered targets. Requires Prometheus v2.45 for endpoints created by the operator. + ## + attachMetadata: + node: false + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## If true, Prometheus use (respect) labels provided by exporter. + ## + honorLabels: true + + ## If true, Prometheus ingests metrics with timestamp provided by exporter. If false, Prometheus ingests metrics with timestamp of scrape. + ## + honorTimestamps: true + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Enable scraping the kubelet over https. For requirements to enable this see + ## https://github.com/prometheus-operator/prometheus-operator/issues/926 + ## + https: true + + ## Enable scraping /metrics/cadvisor from kubelet's service + ## + cAdvisor: true + + ## Enable scraping /metrics/probes from kubelet's service + ## + probes: true + + ## Enable scraping /metrics/resource from kubelet's service + ## This is disabled by default because container metrics are already exposed by cAdvisor + ## + resource: false + # From kubernetes 1.18, /metrics/resource/v1alpha1 renamed to /metrics/resource + resourcePath: "/metrics/resource/v1alpha1" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + cAdvisorMetricRelabelings: + # Drop less useful container CPU metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_cpu_(cfs_throttled_seconds_total|load_average_10s|system_seconds_total|user_seconds_total)' + # Drop less useful container / always zero filesystem metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_fs_(io_current|io_time_seconds_total|io_time_weighted_seconds_total|reads_merged_total|sector_reads_total|sector_writes_total|writes_merged_total)' + # Drop less useful / always zero container memory metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_memory_(mapped_file|swap)' + # Drop less useful container process metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_(file_descriptors|tasks_state|threads_max)' + # Drop container spec metrics that overlap with kube-state-metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_spec.*' + # Drop cgroup metrics with no pod. + - sourceLabels: [id, pod] + action: drop + regex: '.+;' + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesMetricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + cAdvisorRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + resourceRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + relabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kube controller manager +## +kubeControllerManager: + enabled: false + + ## If your kube controller manager is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeControllerManager.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.22. + ## + port: null + targetPort: null + # selector: + # component: kube-controller-manager + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-controller-manager + + ## Enable scraping kube-controller-manager over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + # Skip TLS certificate validation when scraping + insecureSkipVerify: null + + # Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping coreDns. Use either this or kubeDns +## +coreDns: + enabled: true + service: + enabled: true + port: 9153 + targetPort: 9153 + # selector: + # k8s-app: kube-dns + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kubeDns. Use either this or coreDns +## +kubeDns: + enabled: false + service: + dnsmasq: + port: 10054 + targetPort: 10054 + skydns: + port: 10055 + targetPort: 10055 + # selector: + # k8s-app: kube-dns + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqMetricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqRelabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping etcd +## +kubeEtcd: + enabled: false + + ## If your etcd is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## Etcd service. If using kubeEtcd.endpoints only the port and targetPort are used + ## + service: + enabled: true + port: 2381 + targetPort: 2381 + # selector: + # component: etcd + + ## Configure secure access to the etcd cluster by loading a secret into prometheus and + ## specifying security configuration below. For example, with a secret named etcd-client-cert + ## + ## serviceMonitor: + ## scheme: https + ## insecureSkipVerify: false + ## serverName: localhost + ## caFile: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + ## certFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client + ## keyFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + ## + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + scheme: http + insecureSkipVerify: false + serverName: "" + caFile: "" + certFile: "" + keyFile: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: etcd + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube scheduler +## +kubeScheduler: + enabled: false + + ## If your kube scheduler is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeScheduler.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.23. + ## + port: null + targetPort: null + # selector: + # component: kube-scheduler + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + ## Enable scraping kube-scheduler over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-scheduler + + ## Skip TLS certificate validation when scraping + insecureSkipVerify: null + + ## Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube proxy +## +kubeProxy: + enabled: false + + ## If your kube proxy is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + service: + enabled: true + port: 10249 + targetPort: 10249 + # selector: + # k8s-app: kube-proxy + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-proxy + + ## Enable scraping kube-proxy over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## + https: false + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube state metrics +## +kubeStateMetrics: + enabled: true + +## Configuration for kube-state-metrics subchart +## +kube-state-metrics: + namespaceOverride: "" + rbac: + create: true + releaseLabel: true + prometheus: + monitor: + enabled: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape Timeout. If not set, the Prometheus default scrape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + # Keep labels from scraped data, overriding server-side labels + ## + honorLabels: true + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + selfMonitor: + enabled: false + +## Deploy node exporter as a daemonset to all nodes +## +nodeExporter: + enabled: true + operatingSystems: + linux: + enabled: true + darwin: + enabled: true + + ## ForceDeployDashboard Create dashboard configmap even if nodeExporter deployment has been disabled + ## + forceDeployDashboards: false + +## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + namespaceOverride: "" + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + releaseLabel: true + extraArgs: + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ + service: + portName: http-metrics + prometheus: + monitor: + enabled: true + + jobLabel: jobLabel + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__] + # separator: ; + # regex: ^node_mountstats_nfs_(event|operations|transport)_.+ + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Manages Prometheus and Alertmanager components +## +prometheusOperator: + enabled: true + + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-operator' by default + fullnameOverride: "" + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Strategy of the deployment + ## + strategy: {} + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # Users who are deploying this chart in GKE private clusters will need to add firewall rules to expose this port for admissions webhooks + internalPort: 8443 + + ## Admission webhook support for PrometheusRules resources added in Prometheus Operator 0.30 can be enabled to prevent incorrectly formatted + ## rules from making their way into prometheus and potentially preventing the container from starting + admissionWebhooks: + ## Valid values: Fail, Ignore, IgnoreOnInstallOnly + ## IgnoreOnInstallOnly - If Release.IsInstall returns "true", set "Ignore" otherwise "Fail" + failurePolicy: "" + ## The default timeoutSeconds is 10 and the maximum value is 30. + timeoutSeconds: 10 + enabled: true + ## A PEM encoded CA bundle which will be used to validate the webhook's server certificate. + ## If unspecified, system trust roots on the apiserver are used. + caBundle: "" + ## If enabled, generate a self-signed certificate, then patch the webhook configurations with the generated data. + ## On chart upgrades (or if the secret exists) the cert will not be re-generated. You can use this to provide your own + ## certs ahead of time if you wish. + ## + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + + namespaceSelector: {} + + deployment: + enabled: false + + ## Number of replicas + ## + replicas: 1 + + ## Strategy of the deployment + ## + strategy: {} + + # Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: {} + # maxUnavailable: 1 + # minAvailable: 1 + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # The default webhook port is 10250 in order to work out-of-the-box in GKE private clusters and avoid adding firewall rules. + internalPort: 10250 + + ## Service account for Prometheus Operator Webhook to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + automountServiceAccountToken: false + create: true + name: "" + + ## Configuration for Prometheus operator Webhook service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 31080 + + nodePortTls: 31443 + + ## Additional ports to open for Prometheus operator Webhook service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator webhook deployment + # ## + labels: {} + + ## Annotations to add to the operator webhook deployment + ## + annotations: {} + + ## Labels to add to the operator webhook pod + ## + podLabels: {} + + ## Annotations to add to the operator webhook pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + ## Prometheus-operator webhook image + ## + image: + registry: quay.io + repository: rancher/mirrored-prometheus-operator-admission-webhook + # if not set appVersion field from Chart.yaml is used + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + + ## Liveness probe + ## + livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Readiness probe + ## + readinessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Resource limits & requests + ## + resources: {} + # limits: + # cpu: 200m + # memory: 200Mi + # requests: + # cpu: 100m + # memory: 100Mi + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + patch: + enabled: true + image: + repository: rancher/mirrored-ingress-nginx-kube-webhook-certgen + tag: v20221220-controller-v1.5.1-58-g787ea74b6 + sha: "" + pullPolicy: IfNotPresent + resources: {} + ## Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + podAnnotations: {} + nodeSelector: {} + affinity: {} + tolerations: [] + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 2000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + seccompProfile: + type: RuntimeDefault + + # Security context for create job container + createSecretJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Security context for patch job container + patchWebhookJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Use certmanager to generate webhook certs + certManager: + enabled: false + # self-signed root certificate + rootCert: + duration: "" # default to be 5y + admissionCert: + duration: "" # default to be 1y + # issuerRef: + # name: "issuer" + # kind: "ClusterIssuer" + + ## Namespaces to scope the interaction of the Prometheus Operator and the apiserver (allow list). + ## This is mutually exclusive with denyNamespaces. Setting this to an empty object will disable the configuration + ## + namespaces: {} + # releaseNamespace: true + # additional: + # - kube-system + + ## Namespaces not to scope the interaction of the Prometheus Operator (deny list). + ## + denyNamespaces: [] + + ## Filter namespaces to look for prometheus-operator custom resources + ## + alertmanagerInstanceNamespaces: [] + alertmanagerConfigNamespaces: [] + prometheusInstanceNamespaces: [] + thanosRulerInstanceNamespaces: [] + + ## The clusterDomain value will be added to the cluster.peer option of the alertmanager. + ## Without this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated:9094 (default value) + ## With this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated.namespace.svc.cluster-domain:9094 + ## + # clusterDomain: "cluster.local" + + networkPolicy: + ## Enable creation of NetworkPolicy resources. + ## + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # egress: + + ## match labels used in selector + # matchLabels: {} + + ## Service account for Prometheus Operator to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + + ## Configuration for Prometheus operator service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30080 + + nodePortTls: 30443 + + ## Additional ports to open for Prometheus operator service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator deployment + # ## + labels: {} + + ## Annotations to add to the operator deployment + ## + annotations: {} + + ## Labels to add to the operator pod + ## + podLabels: {} + + ## Annotations to add to the operator pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + kubeletService: + ## If true, the operator will create and maintain a service for scraping kubelets + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/helm/prometheus-operator/README.md + ## + enabled: true + namespace: kube-system + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-kubelet' by default + name: "" + + ## Create a servicemonitor for the operator + ## + serviceMonitor: + ## If true, create a serviceMonitor for prometheus operator + ## + selfMonitor: true + + ## Labels for ServiceMonitor + additionalLabels: {} + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape timeout. If not set, the Prometheus default scrape timeout is used. + scrapeTimeout: "" + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Resource limits & requests + ## + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + + ## Operator Environment + ## env: + ## VARIABLE: value + env: + GOGC: "30" + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Enable vertical pod autoscaler support for prometheus-operator + verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + updateMode: Auto + + ## Prometheus-operator image + ## + image: + repository: rancher/mirrored-prometheus-operator-prometheus-operator + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Prometheus image to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImage: prometheus/prometheus + + ## Prometheus image registry to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImageRegistry: quay.io + + ## Alertmanager image to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImage: prometheus/alertmanager + + ## Alertmanager image registry to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImageRegistry: quay.io + + ## Prometheus-config-reloader + ## + prometheusConfigReloader: + image: + repository: rancher/mirrored-prometheus-operator-prometheus-config-reloader + tag: v0.72.0 + sha: "" + + # add prometheus config reloader liveness and readiness probe. Default: false + enableProbe: false + + # resource config for prometheusConfigReloader + resources: {} + # requests: + # cpu: 200m + # memory: 50Mi + # limits: + # cpu: 200m + # memory: 50Mi + + ## Thanos side-car image when configured + ## + thanosImage: + repository: rancher/mirrored-thanos-thanos + tag: v0.34.1 + sha: "" + + ## Set a Label Selector to filter watched prometheus and prometheusAgent + ## + prometheusInstanceSelector: "" + + ## Set a Label Selector to filter watched alertmanager + ## + alertmanagerInstanceSelector: "" + + ## Set a Label Selector to filter watched thanosRuler + thanosRulerInstanceSelector: "" + + ## Set a Field Selector to filter watched secrets + ## + secretFieldSelector: "type!=kubernetes.io/dockercfg,type!=kubernetes.io/service-account-token,type!=helm.sh/release.v1" + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## Additional volumes + ## + extraVolumes: [] + + ## Additional volume mounts + ## + extraVolumeMounts: [] + +## Deploy a Prometheus instance +## +prometheus: + enabled: true + + ## Toggle prometheus into agent mode + ## Note many of features described below (e.g. rules, query, alerting, remote read, thanos) will not work in agent mode. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/designs/prometheus-agent.md + ## + agentMode: false + + ## Annotations for Prometheus + ## + annotations: {} + + ## Configure network policy for the prometheus + networkPolicy: + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # endpointSelector: + # egress: + # ingress: + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app: prometheus + + ## Service account for Prometheuses to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + # Service for thanos service discovery on sidecar + # Enable this can make Thanos Query can use + # `--store=dnssrv+_grpc._tcp.${kube-prometheus-stack.fullname}-thanos-discovery.${namespace}.svc.cluster.local` to discovery + # Thanos sidecar on prometheus nodes + # (Please remember to change ${kube-prometheus-stack.fullname} and ${namespace}. Not just copy and paste!) + thanosService: + enabled: false + annotations: {} + labels: {} + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## ClusterIP to assign + # Default is to make this a headless service ("None") + clusterIP: "None" + + ## Port to expose on each node, if service type is NodePort + ## + nodePort: 30901 + httpNodePort: 30902 + + # ServiceMonitor to scrape Sidecar metrics + # Needs thanosService to be enabled as well + thanosServiceMonitor: + enabled: false + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + metricRelabelings: [] + + ## relabel configs to apply to samples before ingestion. + relabelings: [] + + # Service for external access to sidecar + # Enabling this creates a service to expose thanos-sidecar outside the cluster. + thanosServiceExternal: + enabled: false + annotations: {} + labels: {} + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: LoadBalancer + + ## Port to expose on each node + ## + nodePort: 30901 + httpNodePort: 30902 + + ## Configuration for Prometheus service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Prometheus Service to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 8081 + + ## Port for Prometheus Reloader to listen on + ## + reloaderWebPort: 8080 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30090 + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Additional ports to open for Prometheus service + ## + additionalPorts: [] + # additionalPorts: + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + ## Consider that all endpoints are considered "ready" even if the Pods themselves are not + ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec + publishNotReadyAddresses: false + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Configuration for creating a separate Service for each statefulset Prometheus replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Prometheus Service per replica to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30091 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configure pod disruption budgets for Prometheus + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + # Ingress exposes thanos sidecar outside the cluster + thanosIngress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + servicePort: 10901 + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30901 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - thanos-gateway.domain.com + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Thanos Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: thanos-gateway-tls + # hosts: + # - thanos-gateway.domain.com + # + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Redirect ingress to an additional defined port on the service + # servicePort: 8081 + + ## Hostnames. + ## Must be provided if Ingress is enabled. + ## + # hosts: + # - prometheus.domain.com + hosts: [] + + ## Paths to use for ingress rules - one path should match the prometheusSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Prometheus Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: prometheus-general-tls + # hosts: + # - prometheus.example.com + + ## Configuration for creating an Ingress that will map to each Prometheus replica service + ## prometheus.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for Prometheus per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "prometheus" + + ## Configure additional options for default pod security policy for Prometheus + ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + allowedCapabilities: [] + allowedHostPaths: [] + volumes: [] + + serviceMonitor: + ## If true, create a serviceMonitor for prometheus + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting prometheusSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec + ## + prometheusSpec: + ## If true, pass --storage.tsdb.max-block-duration=2h to prometheus. This is already done if using Thanos + ## + disableCompaction: false + ## APIServerConfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#apiserverconfig + ## + apiserverConfig: {} + + ## Allows setting additional arguments for the Prometheus container + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.Prometheus + additionalArgs: [] + + ## Interval between consecutive scrapes. + ## Defaults to 30s. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/release-0.44/pkg/prometheus/promcfg.go#L180-L183 + ## + scrapeInterval: "30s" + + ## Number of seconds to wait for target to respond before erroring + ## + # scrapeTimeout: "30s" + + ## Interval between consecutive evaluations. + ## + evaluationInterval: "30s" + + ## ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. + ## + listenLocal: false + + ## EnableAdminAPI enables Prometheus the administrative HTTP API which includes functionality such as deleting time series. + ## This is disabled by default. + ## ref: https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis + ## + enableAdminAPI: false + + ## Sets version of Prometheus overriding the Prometheus version as derived + ## from the image tag. Useful in cases where the tag does not follow semver v2. + version: "" + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#webtlsconfig + web: {} + + ## Exemplars related settings that are runtime reloadable. + ## It requires to enable the exemplar storage feature to be effective. + exemplars: "" + ## Maximum number of exemplars stored in memory for all series. + ## If not set, Prometheus uses its default value. + ## A value of zero or less than zero disables the storage. + # maxSize: 100000 + + # EnableFeatures API enables access to Prometheus disabled features. + # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/ + enableFeatures: [] + # - exemplar-storage + + ## Image of Prometheus. + ## + image: + repository: rancher/mirrored-prometheus-prometheus + tag: v2.50.1 + sha: "" + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: prometheus + + ## Alertmanagers to which alerts will be sent + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerendpoints + ## + ## Default configuration will connect to the alertmanager deployed as part of this release + ## + alertingEndpoints: [] + # - name: "" + # namespace: "" + # port: http + # scheme: http + # pathPrefix: "" + # tlsConfig: {} + # bearerTokenFile: "" + # apiVersion: v2 + + ## External labels to add to any time series or alerts when communicating with external systems + ## + externalLabels: {} + + ## enable --web.enable-remote-write-receiver flag on prometheus-server + ## + enableRemoteWriteReceiver: false + + ## Name of the external label used to denote replica name + ## + replicaExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote replica name + ## + replicaExternalLabelNameClear: false + + ## Name of the external label used to denote Prometheus instance name + ## + prometheusExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote Prometheus instance name + ## + prometheusExternalLabelNameClear: false + + ## External URL at which Prometheus will be reachable. + ## + externalUrl: "" + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not + ## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated + ## with the new list of secrets. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The ConfigMaps are mounted into /etc/prometheus/configmaps/. + ## + configMaps: [] + + ## QuerySpec defines the query command line flags when starting Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#queryspec + ## + query: {} + + ## If nil, select own namespace. Namespaces to be selected for PrometheusRules discovery. + ruleNamespaceSelector: {} + ## Example which selects PrometheusRules in namespaces with label "prometheus" set to "somelabel" + # ruleNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.ruleSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the PrometheusRule resources created + ## + ruleSelectorNilUsesHelmValues: false + + ## PrometheusRules to be selected for target discovery. + ## If {}, select all PrometheusRules + ## + ruleSelector: {} + ## Example which select all PrometheusRules resources + ## with label "prometheus" with values any of "example-rules" or "example-rules-2" + # ruleSelector: + # matchExpressions: + # - key: prometheus + # operator: In + # values: + # - example-rules + # - example-rules-2 + # + ## Example which select all PrometheusRules resources with label "role" set to "example-rules" + # ruleSelector: + # matchLabels: + # role: example-rules + + ## If true, a nil or {} value for prometheus.prometheusSpec.serviceMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the servicemonitors created + ## + serviceMonitorSelectorNilUsesHelmValues: false + + ## ServiceMonitors to be selected for target discovery. + ## If {}, select all ServiceMonitors + ## + serviceMonitorSelector: {} + ## Example which selects ServiceMonitors with label "prometheus" set to "somelabel" + # serviceMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for ServiceMonitor discovery. + ## + serviceMonitorNamespaceSelector: {} + ## Example which selects ServiceMonitors in namespaces with label "prometheus" set to "somelabel" + # serviceMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.podMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the podmonitors created + ## + podMonitorSelectorNilUsesHelmValues: false + + ## PodMonitors to be selected for target discovery. + ## If {}, select all PodMonitors + ## + podMonitorSelector: {} + ## Example which selects PodMonitors with label "prometheus" set to "somelabel" + # podMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for PodMonitor discovery. + podMonitorNamespaceSelector: {} + ## Example which selects PodMonitor in namespaces with label "prometheus" set to "somelabel" + # podMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.probeSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the probes created + ## + probeSelectorNilUsesHelmValues: true + + ## Probes to be selected for target discovery. + ## If {}, select all Probes + ## + probeSelector: {} + ## Example which selects Probes with label "prometheus" set to "somelabel" + # probeSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for Probe discovery. + probeNamespaceSelector: {} + ## Example which selects Probe in namespaces with label "prometheus" set to "somelabel" + # probeNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.scrapeConfigSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the scrapeConfigs created + ## + scrapeConfigSelectorNilUsesHelmValues: true + + ## scrapeConfigs to be selected for target discovery. + ## If {}, select all scrapeConfigs + ## + scrapeConfigSelector: {} + ## Example which selects scrapeConfigs with label "prometheus" set to "somelabel" + # scrapeConfigSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for scrapeConfig discovery. + scrapeConfigNamespaceSelector: {} + ## Example which selects scrapeConfig in namespaces with label "prometheus" set to "somelabel" + # scrapeConfigNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## How long to retain metrics + ## + retention: 10d + + ## Maximum size of metrics + ## + retentionSize: "" + + ## Allow out-of-order/out-of-bounds samples ingested into Prometheus for a specified duration + ## See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb + tsdb: + outOfOrderTimeWindow: 0s + + ## Enable compression of the write-ahead log using Snappy. + ## + walCompression: true + + ## If true, the Operator won't process any Prometheus configuration changes + ## + paused: false + + ## Number of replicas of each shard to deploy for a Prometheus deployment. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## + replicas: 1 + + ## EXPERIMENTAL: Number of shards to distribute targets onto. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. + ## Increasing shards will not reshard data either but it will continue to be available from the same instances. + ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. + ## Sharding is done on the content of the `__address__` target meta-label. + ## + shards: 1 + + ## Log level for Prometheus be configured in + ## + logLevel: info + + ## Log format for Prometheus be configured in + ## + logFormat: logfmt + + ## Prefix used to register routes, overriding externalUrl route. + ## Useful for proxies that rewrite URLs. + ## + routePrefix: / + + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the prometheus pods. + ## + podMetadata: {} + # labels: + # app: prometheus + # k8s-app: prometheus + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the prometheus instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## The remote_read spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotereadspec + remoteRead: [] + # - url: http://remote1/read + ## additionalRemoteRead is appended to remoteRead + additionalRemoteRead: [] + + ## The remote_write spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotewritespec + remoteWrite: [] + # - url: http://remote1/push + ## additionalRemoteWrite is appended to remoteWrite + additionalRemoteWrite: [] + + ## Enable/Disable Grafana dashboards provisioning for prometheus remote write feature + remoteWriteDashboards: false + + ## Resource limits & requests + ## + resources: + limits: + memory: 3000Mi + cpu: 1000m + requests: + memory: 750Mi + cpu: 750m + + ## Prometheus StorageSpec for persistent data + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storageSpec: {} + ## Using PersistentVolumeClaim + ## + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + ## Using tmpfs volume + ## + # emptyDir: + # medium: Memory + + # Additional volumes on the output StatefulSet definition. + volumes: + - name: nginx-home + emptyDir: {} + - name: prometheus-nginx + configMap: + name: prometheus-nginx-proxy-config + defaultMode: 438 + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations + ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form + ## as specified in the official Prometheus documentation: + ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are + ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility + ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible + ## scrape configs are going to break Prometheus after the upgrade. + ## AdditionalScrapeConfigs can be defined as a list or as a templated string. + ## + ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the + ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes + ## + additionalScrapeConfigs: [] + # - job_name: kube-etcd + # kubernetes_sd_configs: + # - role: node + # scheme: https + # tls_config: + # ca_file: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + # cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client + # key_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + # relabel_configs: + # - action: labelmap + # regex: __meta_kubernetes_node_label_(.+) + # - source_labels: [__address__] + # action: replace + # targetLabel: __address__ + # regex: ([^:;]+):(\d+) + # replacement: ${1}:2379 + # - source_labels: [__meta_kubernetes_node_name] + # action: keep + # regex: .*mst.* + # - source_labels: [__meta_kubernetes_node_name] + # action: replace + # targetLabel: node + # regex: (.*) + # replacement: ${1} + # metric_relabel_configs: + # - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone) + # action: labeldrop + # + ## If scrape config contains a repetitive section, you may want to use a template. + ## In the following example, you can see how to define `gce_sd_configs` for multiple zones + # additionalScrapeConfigs: | + # - job_name: "node-exporter" + # gce_sd_configs: + # {{range $zone := .Values.gcp_zones}} + # - project: "project1" + # zone: "{{$zone}}" + # port: 9100 + # {{end}} + # relabel_configs: + # ... + + + ## If additional scrape configurations are already deployed in a single secret file you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalScrapeConfigs + additionalScrapeConfigsSecret: {} + # enabled: false + # name: + # key: + + ## additionalPrometheusSecretsAnnotations allows to add annotations to the kubernetes secret. This can be useful + ## when deploying via spinnaker to disable versioning on the secret, strategy.spinnaker.io/versioned: 'false' + additionalPrometheusSecretsAnnotations: {} + + ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified + ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#. + ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. + ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this + ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release + ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. + ## + additionalAlertManagerConfigs: [] + # - consul_sd_configs: + # - server: consul.dev.test:8500 + # scheme: http + # datacenter: dev + # tag_separator: ',' + # services: + # - metrics-prometheus-alertmanager + + ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertManagerConfigs + additionalAlertManagerConfigsSecret: {} + # name: + # key: + # optional: false + + ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended + ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the + ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. + ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the + ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel + ## configs are going to break Prometheus after the upgrade. + ## + additionalAlertRelabelConfigs: [] + # - separator: ; + # regex: prometheus_replica + # replacement: $1 + # action: labeldrop + + ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertRelabelConfigs + additionalAlertRelabelConfigsSecret: {} + # name: + # key: + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. + ## https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. + ## This section is experimental, it may change significantly without deprecation notice in any release. + ## This is experimental and may change significantly without backward compatibility in any release. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#thanosspec + ## + thanos: {} + # secretProviderClass: + # provider: gcp + # parameters: + # secrets: | + # - resourceName: "projects/$PROJECT_ID/secrets/testsecret/versions/latest" + # fileName: "objstore.yaml" + ## ObjectStorageConfig configures object storage in Thanos. + # objectStorageConfig: + # # use existing secret, if configured, objectStorageConfig.secret will not be used + # existingSecret: {} + # # name: "" + # # key: "" + # # will render objectStorageConfig secret data and configure it to be used by Thanos custom resource, + # # ignored when prometheusspec.thanos.objectStorageConfig.existingSecret is set + # # https://thanos.io/tip/thanos/storage.md/#s3 + # secret: {} + # # type: S3 + # # config: + # # bucket: "" + # # endpoint: "" + # # region: "" + # # access_key: "" + # # secret_key: "" + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. + ## if using proxy extraContainer update targetPort with proxy container port + containers: | + - name: prometheus-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.proxy.image.repository }}:{{ .Values.prometheus.prometheusSpec.proxy.image.tag }}" + ports: + - containerPort: 8081 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: prometheus-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## PortName to use for Prometheus. + ## + portName: "http-web" + + ## ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files + ## on the file system of the Prometheus container e.g. bearer token files. + arbitraryFSAccessThroughSMs: false + + ## OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor + ## or PodMonitor to true, this overrides honor_labels to false. + overrideHonorLabels: false + + ## OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs. + overrideHonorTimestamps: false + + ## When ignoreNamespaceSelectors is set to true, namespaceSelector from all PodMonitor, ServiceMonitor and Probe objects will be ignored, + ## they will only discover targets within the namespace of the PodMonitor, ServiceMonitor and Probe object, + ## and servicemonitors will be installed in the default service namespace. + ## Defaults to false. + ignoreNamespaceSelectors: true + + ## EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. + ## The label value will always be the namespace of the object that is being created. + ## Disabled by default + enforcedNamespaceLabel: "" + + ## PrometheusRulesExcludedFromEnforce - list of prometheus rules to be excluded from enforcing of adding namespace labels. + ## Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair + ## Deprecated, use `excludedFromEnforcement` instead + prometheusRulesExcludedFromEnforce: [] + + ## ExcludedFromEnforcement - list of object references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + ## to be excluded from enforcing a namespace label of origin. + ## Works only if enforcedNamespaceLabel set to true. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#objectreference + excludedFromEnforcement: [] + + ## QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, + ## and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such + ## as /dev/stdout to log querie information to the default Prometheus log stream. This is only available in versions + ## of Prometheus >= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/) + queryLogFile: false + + # Use to set global sample_limit for Prometheus. This act as default SampleLimit for ServiceMonitor or/and PodMonitor. + # Set to 'false' to disable global sample_limit. or set to a number to override the default value. + sampleLimit: false + + # EnforcedKeepDroppedTargetsLimit defines on the number of targets dropped by relabeling that will be kept in memory. + # The value overrides any spec.keepDroppedTargets set by ServiceMonitor, PodMonitor, Probe objects unless spec.keepDroppedTargets + # is greater than zero and less than spec.enforcedKeepDroppedTargets. 0 means no limit. + enforcedKeepDroppedTargets: 0 + + ## EnforcedSampleLimit defines global limit on number of scraped samples that will be accepted. This overrides any SampleLimit + ## set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the SampleLimit to keep overall + ## number of samples/series under the desired limit. Note that if SampleLimit is lower that value will be taken instead. + enforcedSampleLimit: false + + ## EnforcedTargetLimit defines a global limit on the number of scraped targets. This overrides any TargetLimit set + ## per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the TargetLimit to keep the overall + ## number of targets under the desired limit. Note that if TargetLimit is lower, that value will be taken instead, except + ## if either value is zero, in which case the non-zero value will be used. If both values are zero, no limit is enforced. + enforcedTargetLimit: false + + + ## Per-scrape limit on number of labels that will be accepted for a sample. If more than this number of labels are present + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelLimit: false + + ## Per-scrape limit on length of labels name that will be accepted for a sample. If a label name is longer than this number + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelNameLengthLimit: false + + ## Per-scrape limit on length of labels value that will be accepted for a sample. If a label value is longer than this + ## number post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus + ## versions 2.27.0 and newer. + enforcedLabelValueLengthLimit: false + + ## AllowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. This is still experimental + ## in Prometheus so it may change in any upcoming release. + allowOverlappingBlocks: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + # Use the host's network namespace if true. Make sure to understand the security implications if you want to enable it. + # When hostNetwork is enabled, this will set dnsPolicy to ClusterFirstWithHostNet automatically. + hostNetwork: false + + # HostAlias holds the mapping between IP and hostnames that will be injected + # as an entry in the pod’s hosts file. + hostAliases: [] + # - ip: 10.10.0.100 + # hostnames: + # - a1.app.local + # - b1.app.local + + ## TracingConfig configures tracing in Prometheus. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheustracingconfig + tracingConfig: {} + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use prometheus.prometheusSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## Defines the maximum time that the `prometheus` container's startup probe + ## will wait before being considered failed. The startup probe will return + ## success after the WAL replay is complete. If set, the value should be + ## greater than 60 (seconds). Otherwise it will be equal to 600 seconds (15 + ## minutes). + maximumStartupDurationSeconds: 0 + + additionalRulesForClusterRole: [] + # - apiGroups: [ "" ] + # resources: + # - nodes/proxy + # verbs: [ "get", "list", "watch" ] + + additionalServiceMonitors: [] + ## Name of the ServiceMonitor to create + ## + # - name: "" + + ## Additional labels to set used for the ServiceMonitorSelector. Together with standard labels from + ## the chart + ## + # additionalLabels: {} + + ## Service label for use in assembling a job name of the form

+8LabsK zE!-KkgVAa$fWlMPq$ScIl*oMT0{ui$MhTEvFY%suJS8=zmKq%??P|65bD_%d-nQd# zg?Db8NJgkS`Mpx|A(3KURuEAlW$<1^KVkq;N>D%ICez(8_thtwaR6LEj$fjpD-5ah z0Ac`f^A~GGKmcNEmd<`foF;n7APNzgMbe2)S(fe@LCr!_o6)Ss_6Y$S2h%kv?$=xRtdJ>x_0pVqcGLIhNBw@ zmEHesK%y0$;=Sca@+#pK`<c)D~n<&-w??KWAo$Ru&?@a?+nWyt3yRyWH`!C0@nj>N_EU?t{ z9`tJ{!KEDiar9W2)HspizlZWW4Emo~Z_xI!*zmSi)cY=MQqG|!N)}w(A5r@okkoUG z`^98R7E#{okin$wi%}BNnX*()iKOQuv*0C7$4L@X!I}-R{aLJ;isgVd9Vz9A>J)kw zmNM0gWzEJDsK+g9n-Wt(mSd~yVoyupA?dde(J0NLiE-~;DmlUmchw}rOq#dVL5FAO zT`Z7)xlupl6tL*p@Xkn9BUtKwH#P-k^edCDC)x)v||VFJIo4)SNZdVGVss&?Ey;DDkng0DPE7b`zH z_j0(pfb0?fu()b)`76rI^Vm9#_pACiF5zk~zo!{Z?XRIPHYI?Ae5jnVbJ*c}25`WU zG4edvhD(w)C;0Dg9t%Jvpc<@!K^>Nu zSZ?OhZh4{EyiMR5fh#v%BmdN*Y8_5ey#EmZHMrw>X-K=?Z#RXBzW%K(4sDx!#sGR+ z=LdQnr<=?hCQN$+E~=hzwP%B&H2pN!!~jyS{D;MwXPgrI>x z*T}y~KxCZ>LZkd{OppRbcw!72$;}oiMkzG_v1v!kIa{P35;R`yBETw4kIJ1RMARwg zuQ-GZBOTUuVWLZU1&QI>%jDAbCZnpZ0+Z3bs(lSYjEWe+;0OV;LJ8HA?eIiW3_9Rp zK>Ql8_|91Q&NJ$;xByT9C;;RD@~jZ~+%0Mu36d_Ea`5s=zO?-)0OS)w-m4RVg~oI& ze>d2sDvrrw6(vLh1+Vc1u03HZ5v^j7HmAuE_6umf#|kehys6JfK71dlpx z<8sl9#vC*Rk1#9*&V1*7i-6^k<+(8Byxyn}O$Pu=jw{`Qk0GL#KpJ$!z)d+kxU(F2 zk5~B9C1&1x*EYp%DbUX&lF8d+y-{R)EB$4P5R(9ovN2>vm0!P(@%q$}C_&Ra85i}g z0UhDfJbO2ZOSUt5qsTdhsIGX`yd74$HN7JMu}**O^U;4ELj3UM_TC3YewdP6sPj_1 z>5>xsyAHs%6D)4*LmoB2Hr}J3OywqE+r%h9TVyz3o1|DSro&Fo-`WLufYY3gC){>u zAXb1bXh^`WNLs!d)Iq;(s5Hru4KhVLAMkh0hsx}CyYF`)7C;8Xfh{UQOBy8y?=*Doxi^X z1gvxQJzJ$n#dG@<8uTT_Z9MPdk?U?*S)~t=GYNC0{=|VdipL;Kk`-LkPm#?X1QKrc^Eb55>GL0J)wqYLO{g*4rhwu6F>ujs( zh@vqTSYyVpmyEdhji#@N^nwewJW9TDhjGWT_zb-7%G~UE|C3U_R?COmIxj!bc>XwS zf(J`X!+;?}nm;k3cYhCut)<)X{nKhl-%eIJHt&K_mzAU#ctld46~rkK{z%d|=XXUS z3ekc*2ik&Mm#vg=vslO4oK3yCRlQDV%RO_qtjlr8M~}7VgKLwh4=p{_&E&<$4`aOX z`j({jg1-Mcw`vr~f|9!>s%xNkL-sBeLLNCU+6E~y*a%7NGf|LRRt>Q;8g`VCChyrm zaEF@sCPlF%F{8)q3wc!aF-nF*U%G$T8)sC;3D~Nem4wQ=r!?125#)*DGfyYT62s^)7ujj(8RL#s;S?(fOJp2>OL+g_$Z6RVm_MYT0> z>nAPzHk|)#P%UZboGS~euE5U*3Wu0VnFHl%i}C2|aq}1JHLD}!nh%j19+zz0R+{>ZTtI}T*+CO39Jlm~eq04^|Gj9E7H+}~qpvEL{Ko2y@8 zt>eZ0y~^~#nmS$Y-O0_6jmWw&I82;~VaBS6kB7kE9bnRE^HPetz1hG`eEo6!C*j!@ zBmDyo_<7IO^x?b^DaP($X!CEMxBnFaNoG*9ud4`e(3YSzbEdaaOuTTKsI%{>0ju|; z{Mq35_37%L;_d@)C6f_~uLQ=E0gWJUlKu&h0hRDuGoX~@ou48C@*=fmRbEyglu3mB zp@f{(Nlh~VI7*FXNDpOktW&wq0Cz$#_{ajAR0~->R^dai2x&c#Mt}mUbdgC!vaB%j z4v+lMVn1?cM}J2~d}>t>>PUiy5ryJe#-rM1v!eaVAon#C3r#WAeF~Lne+Rk6U8N-E zk!)y(nV`Hv(jpk`{Q15)htaQPfB& zxp$SzJF&Oa?V1=bL)7-4+8lVWUp$0?R*4&nVHj6B6RRKD1AIs{7={O^*&o=!lwR@NR}p>6^_ zG5k(*pJMNw02 z<(ublmu+0jKZg35G2t@`I#$Js8A%87P|*&@#GU3 zB|cmzSPcuF*qDoD2k-h56UqosdLm;GTVS0|-JprM<_aR8M`B77srECbKeamX^96I& zS_eCvsL?$^D1CiVB|=-)6;S)oY39Fg^0>|5HGmnMkG}3%mtGTZI8Z8EQz@BxB8!zj z@}Vs7r%b?Ci|;h)W~*K71g#H%v^k#AWVZF6dx>B|DUSC6@|$>|MyQgttx6yC=?eeP zaswkS`n{wlP8u0F{OJ)Qd>9qA45+urB(xQ7cFrwt`6D4l>=}7cQ_|_uY76c#cg)vN z-vF?|bnPnR#cs8fjs$aN^FBJhQGNO! zdFDm^#t#ybD$3O)-)F&;ug!=zj+LkoL5mK+#6S9>g0UZna`PGsDT%S7HsrGprWGFX9@MNSg5Je`>KDf+mLLVD-Beu!es zIGV|ut>+=l7q|vtAsTZUaq47K>q&Q$(ObEV5T-$@RG`^EHzPc%-Rk4J2`@l&FxoZ> zg6|+V9TB^Li;dcG0$wts%BdCF!Sp zvwQ^TVCEs|N1x(cO?I^#dwp{X#`MW8A`>$nObAEU|6#t$O7)e0{xM&Zi>d!GU$KjS z%-5}G+VQt?;K+TsSLoK`U^5xndM~x(?a?Xn%?5n72H;J7G@I8y>MQ;jm+1C`e0$~p zgM4`?b?Nqwz5h4z-Rx}sJ3hrJB*gNZQK!20EL?k0F(QVYrPNK)fl5p2*klurO+OvGA?-%EE(u`jWWC9tO3;i}e`# z(#;bNhlFQWcT;vm5OffTIVbFbBP)Tg1LaTkHi;eG$7LmRp`rgR);o}=nVjqnGsWb? z-kG37JbQwTQl_UydK^+z$i3PBlM)twZ4qrq_g1hfNPSK1X0vff*6$p=O~n2YINhhm z=rQnUZ*44Pps|daAJ@4^UORw7uw>e>uxy4SNscQofSMAIKofw9=nS+<*&mv~(z{Fj z!Xbm*vdcm$hsT;*T&2gcGm{qxDUOg8M%N{x4{Py}#QXgpGt?pb-VHw=&|l4D)}R=Z zkKN(1b)Ur5kIky@jylgYsv6OaBdIEEka9`l7S$hnc#^)D(M)gnvi%R|iTWR$C#0e1 zOQ+daoETP-QA|QNbGXh?{A}6=?YdU{1LgIUzQ7D#ac+WZwY*o>&}de6dTU*A{_o(% zX&iWm0OeTWI;zo9Wp53D3#gC3HX#{UM?g^025J${JZ=jW#-OnxJ*A&Gq0gI4I!cYe zF{r0N4#Fkm4@bFz{9-{QE zJo{2ZE?G349voP{jRn|<$~ct!f#xG(dM07fu>3^=Fx~~6Eqn092n~|27^ zLNp6N7=&^%?`k~8uvG}fNi(xh@+W>h)ocPqEnu-PR%My_?8huIVKWIq;nA2#G0r2q z&+S((;p+PD(O_G$xyw+ms>`jdmKL%uZWgjwGFO==EX`fGIqNzDy;JZpSskgu%GF zVa!lm$UxEOft$}V5LSkM*r8bYeSXq&u?!Qy`%}n3`>5IoZX~4J*Ndg5ylK+**ZtZM zzsI9waxes9dM^pw0N6N3<@#$fa0#v;=W}{b0nBAzFV@s?yWOEhLK{qZT?+@;K&;^O z904*1YWj@oe)~=3zf|iv#`UmUL$Zb{B#1WuhIyJj2FQm)I0sMx7jyij1serdb8xRZpKU*$D^tp*$cDhu%WL*0aLaZjlIUVfj>i+s=it>^#1az2A401@>zRMLWv47=4Ki~&+>?M+>xQyMK<@fsln%-0jCQVVG zoh{9AABH^U1cchi+RhLthq}}XXE5s?6=0l;5I|xM)UP4IZ9_YqzGM!xenMc}*f13y zq_E)Se_DW6i>}MqOP=#I|E|o*36viT;2;FKgz^8u-$D{s;r`$F+u#p>%lmKqEgh9$ zO+QJ((|`XDVXXvf3=*iwD>1>gH%>Y^ezqJkbaJu7UA~)aRi zHYl!%U#dxe<@l?yM3*6lT&^`V5EZE=!+jMc@Xr)JLo{)q;bThaU~LcrC5!3iDN&?R zw|NkE&^S<%x^HX9y1fe`%7sa{2?wnuH|`I!a)OlSVzn{^cwc>k!iGrT+mTlLwa}iH#DOX_e{6 zS(?;HD<8g&A8)UV3nS)R^#n%T%**2V{ctjvA&_~NUqJoT-5k!`WQ23_sy zvAo2rtSGBnwso!E^9)p;`So%O5y6JOSgwJm07A27O}O(t`a^a!&I|#b zfy_N|rUHI+x%h0F2ah0L9~c&A$=Rm7XMmK&jusBY_V1Aynyvb(-jd61Ceo6n9|C9S z6Y3u1L@jkC<3+4gppkB}xv`XP@Oivj31=~6aB3{(GNr9jpA4qmDNs)@d?k6gkH|t< zIA|M+#6)J{E2#J;gha6#GAQ?>uK;@zA^B$if(^z4VC>?ON0t-lki_LioZ!B#qTIU~E9Vbn z({=Ka;skWHa>cnl)$e_@D ztEu}UQBo^_X8w<_=>c|e2}q0{LcQStp=gShl7GlrctZ>uzClvbka}RYSclq2miXG= z-uk8erD~bvFp2m-G_UxN<^_)yef#g47xDk5d6E7fnpepf;yYZM?OMA@Fj-oU%iWCC za)^#+w4d6((5!60!VUaWVNLgvf)U;*GMdgSX=LKHjZMB7` zor}_Se9G^a)QIiM!}#CDCwHR}74bQo5LXDs{}IF)fM}!CAC&3F($x<4nu{`Fm12h9 zI@u6`%wPNOV(lzek8qxBgPP?{LCWAdPW)T( zZIJ)tzvUm{BDv!KNnw;rHNyRopY-D@*N+q~arynHWMaUO_v zvr~92xE!ueqTiYA+svQldV{3K#bZDUM(Cx)_COs`-^_*4fui zl7S3Yd2a#SPYvt3mc(^H_d>W`e|J?RPcQu$mDhauX%(7*U;HT+$!V9Z+#sm7*5w9@wZuT&>`ZO_UI3UgQ~@;YJ<(S)VjDvx$l0PM?>E`jAs-^8}T`l zK5iEC=DUu!5)iiMGpA?whwKeq<+OzC#i>Ii9n#CFT`Sg>oFh{d~=Z6+9+FzXiWepuoPHwO<|or zcOen(>re<=D7>LXO7Q&1`t67;PNG-7h9$DlWfnrP91rQDb&vAtxcO+my6fPCG{1sl zLn7RCXTiVrHK69&8RU_rwmpTUy2*~;a`bY!$Q+m4_6ix$ZzdEORgrtO`?NS=|L9&s zZ7-)o`fUMpoLsf$-;2?dzE1NC`EALStSX(z?nU$`dG2oV*_v6U{AQ4+`F3>Gza2;9oKiR^pv)4ZC>gfs(=LQ|oom47ohQ zJ8rO;-I3T?cYul4m=fN4LQ&=d=#2tJ0qOztA~b6N>W$uB>HtSYsBZyurzzzAMuyiM z;03Z`vPckY=ElG&WDZFpMhJXk8{o8p8~N(0hL0xr6`q+>hL*gn143L-C&Sz>8XB(R z8zrScH}5Du*?tp`Y#U|`<2=$p;feIL&TgU|u6+)R z@WPM^*tXcikAH2At36Z*<<@#=?&tdVifwOVlOF?IO;zM)e9r1AB$F{XBUIQOXy-eL z8#;~!=^{2j=a~C0-d#Dr6>C%2k=t{n+X5>zu1^GK#P(ebt02OBEokm4l3l(7nhQ1& z-yg#q2WUb8@6^>C@Dpyxiq*di))D+@*AYN?ihHlXZGw>_@kg3`e8lTKMx+aJ#6P=s zKy|4RcrsMI3lPy2=6c*Gm8&zz;@S_!rRIiR&oJ z-Bi$V=aE)%N`Rh!KM)bs#_G24bkpQ&CsJI|5^mq~qF+J@?0#5QFTcmIUdo z62k*zN^dg|^;h>gPq_G9b6~iDAM;v1ChfOYs9*eSPyRYK?Jt_bzTWnh`Me)xe`nf0 z$Xdp&p0z}t*8eQ8JL|l~P@6IEa*%T#cHJzKYveBC3^p-Sx{-kDG1k%60;=(HE|Tcg z=9w6!y;8KnGgPa+(pk$raOR{{ZOh@Zp7el>$CLy^mNl>X#7s|ev@t;G?P=@^6zUbD z^KpH}c4~xk&j>uRu;$$~QdMg{AO5-C6VuZ)&DtbcLKG`gaw%xNLX?)`Af%`9E8!;I zT>PID`@pGSfMgA0i}J;K3tBDyBf|94^*S+=WcImp=d@bx%<0yui|^fCvX9f&=6uW?`O^`at)saV;Pb{f;WiyATuqlrxf`T z(9%CqTcz<(00H+JbyEj_l;vNsQm}uxxKvDH{_{>t@ItmYMhvKoAoRUiT=f$u;i!CR zw=*|KAJ*~0|D}8)2|JU71y;{%XD1hET3{AEQndwD#w`i|vo+yZxN6lTuIzE| z*I$6BDP09N^qHGb&<8>``x55hQ@)I=cdYQluM@gVHRUCGaNbO(VHky?)8f&GNifx9 z_p*je7GBc}(Mo2Uu8u0z3yXt(Y{~Awl<#htej;%lV4* zz0}Jbhz{;AnWsTelcs}rdHo*xddMYFDnb2OfIYW1l($ja_+&}mT|$$AxBKVhI51h7 zaes`nnyE!^ic)9dO5yo22pQbTIbKWbox4pMyhBSMY)T5+HFcEkH$s#0PwzBpu9dmw z#LHN(fwl*nom_)o^g|Rp9kDv*Kk#S39%>*_Sbw-VGewwJUK8;t1aX3eq424IP?Vv3 zCTL%tOBPd#x_+49o-(|>@n7T;i*i`l%fB;RoqjBhW-a*ykWV#D|AF7cUb2P0c%C)Y zKiqgd{$7-?^ZjL&>y-XF82MhjAhy&(pyUENC;*W<-P9_hXYj8AEl-Ah z;Kgr+-Ryfna;rf=X_`vN2`(-I4w2B9(De_?&;KMZTV!DievF@C1nmFZ+}| z4LFFcTEe46$r!NM;8U&UUm&JT&If>rN@QoPmSMk}V411j)hsSd^x{@bf?rLhU+y#~ zzyhDE_SMJw%iNR_mHRr_qEI@s{(47zu3mXX`M8Q0hp{x8N$i?D7KEd|H9Wz{7KDE? zoH?DHN0K_OsfPBb@{;KE&mklu{9{ErWG%x-uFY-BTocz2Ni!z;bQq$ua`o9ADa)-; zxN<*P%d#Y%g@0K~DAXLQ)Ux+6loBwg`ihzd;Q+-srM)ZUJL~Ef{<*|pSM(kppN2Gz zpz)-d+ZvTPs~%MFbPc04bz!Qf|3ylk23v(2w0-fB0TZ1m*83q918Q6w) zK5|9nwGpkcJ9FvgcnMh@9Eif8*!-ML9r`hP?Yzd5k8~WkKXimVNjgw|7g^TTO2le; z8TZFu-f6~Q^UU9Iu=RyC9rHTfzCyrH*Db3)HhzUYwEvvh*zEkYs`C7kXKuQ~5Z0C2 zYJuTSeLTE63AuJOZ9VA*KI2FHjKMi|iJ{~seMWW^a%v++h9BCcClgG zDzVt>d!-y}(^5(WM7LS|moNAgzvnDgSUHtzSF63PX(RCE{GTn*_9Zx*GHb0o3ZnyD z7kKJ^JBVOfI0UFi*K75IywZEpA7|(#`+qt^D2}K;2!$Y~Z9MRjv3y_;_d%M~wxQy@ znS2Gai_?doZ}DpQJ~t^|!s22fy5W{9`pLD={oOr4h7(n*bmv=@64JM*vMKnk8)4a_ z*@2#2Y>!DCI!mquw#XcW7R$wuSUlpK=&tuS*x#56%%Hd-ksontw3h21rx4yEz94YK ze``vyq+Vn%+wE@(K5MNQ55k2S)v0{!*@;nc5MZ5Gsd*tGf98Oc_q-Ew`q~I{XIC1e zziMqou|SoLbvKeerF4qtFA3}j;R;u)U@}G?hAq~{W8jgXD7jkUjSbJhXZ+uSmV0(U zbGoJ&-@db!iEdSk*C?;;M^D&^8cFu5os;Oc`X#cu8_&%+m-VQQl!Eu0)2M$vm=xus zkDU>m%saCg>md(f_Bywy9d@)!V|o%I&8&<2>D@)$etQC{QVz+fKdOfWK-P(+Yp~jium<2nIQX+rEuUbtRsQyGlz%R zra2aI$wB!J{Cw;_*DYv~QZ%;VNwJ)qr0tiN#e~yx;;=GghewTytC9qvJrr94RGXLr z137KC#BJR;gdE0=n;>7g!5!3zp*rpIHyGIhY!{VZ&{(|*Bx zZCF7CvW|Ltrf_#MIK#xa#*s)D+6^@bUQR2E{;s|;NVbtgLRgy`4bDA*q=jq<-W_nF zal-A_P2v6h50RGtEh0XM(UbZ?{LyEc7O#q*d`O9(oNh%_P*TMq)okLmDG-(!#fmIT zmNOaLVPNYl9HtuWR^0~rXn8L0sZb+WvaZBFD-BI|T{90jF7yRtOtg}}B}X>yV_7DZ z708I_i%fyMQ6cr>@0><~vL}OTj`&6247jL7QA7%!8?8X!igXhxLO&y#RJ%pOUM~{V z$nU)}_~@3BhB7<%IQfxkBNG$zY(hdCPDls&Wopzpm@~U<;n|} z(mR2F{QlZQNWl06$2P$Ft(4ig(glm=b?`IP_S&~MAWumDr*Vc?J%@LgA~{?@#vw5t zrc$nst5ryOwnPY)G}$y)O_fRFBED`<_{HLch2We*M{zoyIJQ!uDk!3y z3u7}!8{?VM6V`Mgz#{D|a^NsTgG_^^%Q&g>g2I~+*UBW++ZLws326ZlvG_Pu$pa$# z*nFMhJPf>e!ldH;t6x^tpbZ{m59NUEeJM+F{Nqy^#-k9=UPxW3T&GF>H_?`W&TvV) zm$dO(2DzXiP_4_Ig%YcoHV_iG4|!kTf0ZI4iPHfY+^+Q z%~G)HIjUAUZIgnwUNHw?n~ZnTFU?G|xFcsdheufU(0#hX0GRZ;X=UUz@D9ZQJH_ zPkY+7ZQHhO+qOB4X=8fYoVIOm{qMcI`>wuho>S*kWme|NtV%o)zleC8Z>>oAhv=`v zyNnaaTqHa>62S@OaGb*lfh;pW9XQs{XDLQLj6BSjf~= zxU6;V;UNz_d+cghQJWWLDg5wOleC_B;apfG4qBcmfHBA~FPj*O2ca4w6POw`KvPsd z*qE0YadnM@k&2R3PENY(Zr;MO zO1$Ij$o)I4Irw%ZJ&RJ_n}gn}9azrQ>NkkrjFHz?gqV|+d8;_8fbeE5viX*Hm3EU5 zq%Qx!bC61_LSqUSwn@W(xK;_GX_VpBKS}xZ$F4P^DvOFdQCU(OSuGn~t&?XGQ(dp3 zzgx=oAd4d9sYt`=dsq`izB9 zw#>CczLCIZ?o2%XsbBfI+k=xA*43ks-cG%XDRrBMxsQSK1oz$A@?{g!)ZO&JZF2IQ zld*oiq^7Q?!{*nWlfs-4+_Sc4>^xOL;~oho#W5>7i-3>d&R0>e(;R`RGI|H(c7M?y zIbS)V>LLm2{MN<(Rur#bUnAnt89vL;xZixZ5XwH=asm}K0~b9#wM!TI@Q3ZvRqL3i zD^*nH!FPFU0t6*gMW#lIq8o!$^!RAEk@ZT+R|%bgSVVnj;a=V^&h8r=tt*?RI z!rcFr24Ry9Wi<)bG6~i^3Dz_@C` zK=6KD7PiS1`p#~zEA=3sLI?`=;Cel@`Z?-}G65$>j!$p;scS`LO_g!@EVQak3kyMZ z|4yUJ(Y5KUs4L-Su>d9%R*U&uUVD9d0blTAPVWo)TC-ah$ioZzf;sr<-DCAEPwjDC z#op)1ib=7V$!E~0lH*W19~p+bqm9rI1cLjU7Yr_=MaOD(_L4)i>7^7I#f;j5!>rJd zXu-~9cUbqcYJBxB?hLJ#U8S8hS9N83e8Ga=hc%68TRG=EDrYPkXD}n7;j=WRX4x;D zO`H`wYR3mA_M-FO2_b|Ufsr`96anOIV&eWk$VuK|2EYR%YXNAYlN}Xtk6HsFfmi@p zK$ZtPdtoH0$Xq3k6sGypR*7@E)O?&;qbeXmG688=>LFBf~M6j z%MUc&Ed`i95l${4W-WM?=GMcQk<|Nrf(5L$2CV8zZQSUNK90}*Auv9`cl|`RjL^Mr z!(jC&Qs+(6n%HjhL7cS%-x_fqO!fdDg45(h_NF>Npj1(c=$rh6)T)edm~2rpeBPX} zI)QoCI1~ubdR4JHS$ajsluNKTBW1uAeX$IyUKhcuNDuyr6`v{Foa1az8~i@DS3U8o z-LGznc2C=KXM5-7LVC1)L#E$2fjX0<;g0|dIjEpYc{LycfNW}S7n|8vT^kY3USY@l z1#&%S{3{0{%$r2TeF+tx(({rwQZC!84!VsuU8a0Rwdwaeve4KH8F-Z~2j=Fw;kI4G ztH=;xJ_q&6_hk${O6-T@7J(a?E24@}nJQbMR%Bnb>ip6(7z2iZwe$G+%Oh=^Ww*ka zU++)@t+SxZbx9?yl}8f}7k%cIwX%QG_p_ndKc>0M^XLXP*KYhNCDhqJE$}D-g7)xU zBnPm9_M3j7$GPp0b8oyt^>gXodTTFP8CXWzM*@Ju&8w7Zk;R2Z{#ci(wH@D=V1eSC zs`tH!>A^&e!^rr9YpsidSzQHIrlHCRbeRgR&*838Ar}QE>j$(2X$7X{Z|!5{O-+6< zG#__Fg8e=+`B9?2`YmtglZN?;Yuu%HT;_h8fdYn9Oq>NlYBtvx!dNV0OGtV+L|5aM z!i^LBrSv(Z74Cos#J1ja`GPtkbUiG_Plu5(dlku}mV3=Y1YJgLbyn#~|Q(Ww+)}{z%=RBri9B% zu7YdjQYL;?{`Jo2cg?CJDZRF|GmgRUNoIp{^+F|TH-a|f{24Df8XYe;tbZdt^mY)t zD~gODzB4)iOf3<{DlmF=Z!(-c989^D9FO?zSmI36=DhrUyow0cdG{^qnuT%Eo-1d9 zm!9z6u*e_NHE4rz&^`y<-a_qj`(v=bbvG8RRYFmrs4LXpI!uY>hyQncIT|FS3IbW1 ze@ljW+D>!~|ILUkv+zv>#Kx?hc`JM!B9(@8IVH;FZTCmU{a9^5# zl(0nQF`b{H<$F8bk0DK#%&4zUv)PVka)Wha>V6?fk@9 zdad$XB1+U#UmWa*mnFz`RqfrTTJvSKUOEo=Pdn0yg$BUmbp>&DL&s~wu8EsY4meHR|RV|OZ@18AFu1orBWOF z4OGQW()>?V?4v+1N@V2P_FM`-iW8i?6uc3&AJ#v7&+*(1COBCmuq6F8_|dp`vLDq^ z<}84fC4M|dwZw44NH9438nHoo z&kgHXji;lq%`uQP`IG0IP$OhVLAaYskXpsEBli+?aTRxs#`ig|g4_PYYr)L8#Am0y z?;67e`^-=RWya$lCG2w9+$32Ew0cGme>iMTZSfzpPpVda@I97~kd!pkUV`oXp~zis+O=GlA!Q~?HxHa%{dcN9J2=14BabCea=HM5iSeWbIxP2Q0%#L{S)skJ6dm)wQb^f!;n z(;rYw5edgeSxguGRqsD(5VOZZGzUS;%y$W_NLGO|yYd(cQJeN~ArA?*b%|p_n&g?; z`=n4x9o`dpII75rC?-GjVH1i4HFsrn3ideLmOou1(KHu|rU*I(Xz({WHSDMP;aV<{aH{wCn-{9Hn_O0VD#Az7G>8Uo-O+B#YVA>N9ppCYZ|G9T|Py~%DM)^(6 z+LKIW5|73r<>rr6q`7pX(hOQImE_7z5N6NnD6k;dG=-?v=0=lesmdMvd#Lf|`dB z40=f{;vWiB1K#kBzdtWP#?XDHMG5aCrHtPzz@3B{;frGNmg9K0tTcjKXyuTr+ z>yxk*^e8wAdT`^#0&W$eZsh<6AQN=FpbCTqJsBnfig}U@u-Nv6AO^f7mXLeCvp7)5 zgv32faL9UbxX5}O+{8Ub>VTa{$+RCHe1aYU*Rb(`5GZIS%2kYz@vGi{k^g*6e;zG@-e!VlMu+oCp>5;z#G{p2%`9+H4|K7jqzvyU)1C2{*aT!MBS zZh|&&`|ZH(cmA_|J5~aW1U%wzf_A`P$_}Ed_I6ry%-TDMx%d+>wN8}*RhR*I0$hS@ zR9iIJg26gmX%6)&2Y3UXfVvdz^0{%JFlUy$-%_rEI4E?>|R>Lczs-lo9w!_7Sh0C6XWe3_eK+8DGd`khYdfQ zsM7gPe??HdgmtoWiOtlIC6$tAQ-Q$@GS1EEyQO9Ji}-tMQLxz=^|F}>SjCAcQe|;6 z#CFC?T_gQ1gR)1Bj^_e<;?Blu1ov1ixA=~*-{Z5LjmykihyR-D?2N0ikDHqtCo#75 z3bPkgmz+y1`iz?F1^HLUh@N%bkDiGNv@f*Jl$CJQGvxlvHopfau7yZT#MZD$^6z$6 z_4U>asab~!`&C0a*O^b-E6NU3uTIU(`LZ%IdmS#a7ME~ zK(1cy{VFalR{NAkFLU%7yHxjivN|3XJurw=p|m74Skaf4jwe2-4S zcNGfY8NkA$L2xRm4xe(BkRXuE0R~ZIQcqupUxA`3^eLk@jkpxym6QuPA)*RAr!c@w zflEVJ6%{C9CqyvvJITy=G^$GX6)3A(q!lQwx|{0@WzZddVx~u~^}i?Z?Pd{FFkm<$ zq@kbQ%#h&5&r{KQzkB=jcY;Xt1w5z{`qCigO7n{PC$#`gvn zH?D_!mh>6<43zztWZ6-{D@PN!;810;PZ`V^M9RW0`38q1zVM%GfvwF$Cl&IIZc*8Y zUAH;NS%aXo|6ElTSkB7r#&>}nDw3ZvOLK^3Rs4szK|bAUbP%@}#M^aqptl#~v%x;y z<_Y$r@>0{_ZEGQ$+*{Lg(Sp+n>benfYQFmt%bi@W{qa(7_C%?x{(6YvhVJ2#83XnS z$E}+ueyoVbjY~-MVQwKKZJg;1QW2df72t&HeuV35Q$tcLkTRGYt00u?YbK*zK~oVm z#sk*tv)0<#JsR2tAkCt(ZUL)HncSWDQk$>uE#woY;Srm3(W%0IxclXjjAQbXio$+c z9Nhcmg|##C)!o8^X=|5em6AyQ>{U)G#IguNSy?&slFG>}xFwagz#i$9F~E4Z>c1go zl2(6r*8yGSm^flGz$Pg5AWZ_i?@v2B{mq(d=AZSz$3|<5G6*NCJ`ogA>b5b5sn6A@o2n_ z%jB&Sy$@U?U*`?>ES+l=A;H`83eZ_DzYTD52k8WJ+J@X`^y3En(kXroxPb?d0e%gV z0hmDzVk92{%>3E3j{(@p*4rq?&1A74>}gMkg2t~dfLH+T69zlu#wLTiJ?Yb=DW8Yy z6HZA!m;_XUo(Fh<3*dxhoG{@2$PE_T;9-Bwy+-f;vZy+u6{O(^2BaaH5pa$%3|OJ7 zbVq53)<*=eE?4je-~$4{0h%7*|D>V-*9MH4ySFeAj{(;Pr@4LvjX*~OzH|YkbP*sT zYD!n65y1c9kDh+y6x7w#WuSZT@xq`_k5&?d2GPEIDLUX5WFKq{OA&NzHkprQY#D|F z>0{tW@f$psSUALnz+*V!Pci;1p0L~2pWe4E{XMT0A>ZJgq{-&|R#?VtkB*7L(@Nu~ zvfI`_*UjISNH6+*AH~Gp?DIdazwCy--iE)n8?=Bk+n)2e-ft?xJc+&P$G8GM!!QM& zXNpIDm{t4T%oFf}dXE@oAKBJFU+P~8;D4esPV|3dJ6FYs4)7{&@i5E(6N2YyWPZbr zO>MH!ZBc;Sk00Pvr|1iy5B?1Z00DpjFqJ?7Gs%3t>D`Qvy7V?5yvW!8bWp>GXZ9$Z zz6<-gtUL=t`R55_&wy)i5D*F%0tgkwO4+L;h9uUJ?2Q)m0Hzgqo(eFBr|*4U>i^Bb z0@g;hh^drN3l_hgtkKP@}0}tQoKHW835>wH=r*&pQmZK9*FHR<{)!1vwdluf! zRz`>CBzh;EIw6Ow375MY1GwNtc0OMnF8ltx{vZPE$Ac?_T&*R&WZn1VEQfvq4YJ+E z6H$Xd@Vs6N2TBCt37jDhP0MEo2sKzcli=%cuXsVO%939)OVboJcsrAaYgQ(g?#9kv z&bn-^_{y`!<2o0>`eg4_zR`eI*VxipgXOKqXlpPErKid!TJoMvgX>^_)?~*A0Z#n7 z%4KJ4=BKX3Lyx*tD;|WcMIrGa@iNP)DCrbd%x~_iy-!PZKAz>+n^oWxU{GT?bewzq z{Wr&aC-;*!VPuc5A02}K>nOvcmb1X$>KOTJ|H1qujRtbmnC+Z&7f){d(_wZzMvZP5 zI=kg4Umacu>DCn9A-UW~_1yA9@_f&QyqfG}DgRWldp=9-OW44Qv-o!3rZF|b6oq1# zR^Kqs0-a|WRXSFgiFmUw6-D{gfwvCcYr96Xl&Q<}kn#PzYn)(tG}@x(WJ(r+oH z{UiGCG3f>j@?)4xKU0&<0vOs88LU0=+L9QSQ>P$&I%9n_C(HU%eLBB(or*hCwcj^q zJ-C-{(Rr_irzfewKqZ?!52qigE&mJwyR7*Vl*v!~r-mf#(sUK9DN?YId#Wi3Cmo?t zdKO`J74hF4Q~&j)wpPu$^CZRw8CfnL9_&Y+jqj5_GxaxlQe*=1lO}o<%)FW@*gB?n zHiHq{O~+JOl1k%!iT~ESnFvYIQ9fiUphjXzQ~D|-NllDV^@VMD$BC|_fX^GBb=EK> zL~~2OBb(wLpr%PAxdhVwOZJwAo^?(kW%^nHG&;#kv_+soi2uvz^o$5J{r{+CR4JP< z4c$$Ic_CLs{iDl!Cj`{KIWlDW{-b?sR%JBHx{+X*vG|1~s~m_1>E$8l60`OW_y3l> zkx2eW@&+t>AhV_y!l6WD@(WgL@IEcWze#W56Di-V)r@J^HcnFasAI%SKJx%00y46J z5doEoz_7u>U{5m|3Ho54Ob(SI8}<4^4NWHPV0d}bv=^3_=lPKg=o>L(+_ow|m!6qo z7-|w&A14A=p`MGQ_RE(?{z*H^6lVoPJRETg*?%N%YV^>>%v7JUG`=O$4|sWCIvx(Z zzBC8_HfQmssGc-MB((0y3K4qbn`!Hg3Cw$YntE=4Ca500XU)y zFDjv~L}2|DI9{hDkWx@T4$-fL$MV&zoj&n6q{P|v{DXq<}=q`{Nte|HKouHcvm?`|7vZ__kzaHh@%6tD8otrsT z)+1Nad0Br7L*>U`|An{qRTVgkm#D9{+R+ayi<{}H+PXr!=;2v%7#-v!K+7-XU6t~@ zjf$z1?Y!RGaR2|cMFE}JzGgS;E~8DL*-dXVDH&?g`9|?C1cxZ|jm)h)H?f21?H-b0 zGw5gEp!8i8tTAB%(n&PYP4J3^Ty9`Te!#afHAagsl*FL+6G&BMRD2-Fo}j^WT$hxH zKt^qgjzX_Pes}4CsLZ6U`>(br#sAo%Lg!KclLip^f2IMHO8942tpc6@j@@uB)dU*~ z>m7pWaeV*X1yy+J?-+2%4_3EN#mY1|3x~ESrN&Y0h%sy1n@pe{?9UPCe!g7sI1{J;8b25$C^|s7-@+UM(_cLnfXdnoNcLkD} zZibi72!Yc+gwCn%M}+GZDMsHaZ?!T%+;W^%Y0Xb>I;-%_ed1B}%z0zR5#k6_CZ7a~ z^Km#Ti8L$XkLm`)7jQCs$lHls;++wDYXt1w@X0(z${&Zx`GlFU}E9?iz7d#bdxqX>Ut z@?X;|dt)uVJ=VAjG$0OrI3Iui~l zJ}PG76~XgV&WfmQFz+Osvf`sYU)ImVRLOh-BV+-AE_6UGBHn@%Xh3aLD(FFkVk?RYm6Di~ z(y5oH`1sfw6`RjKc zjEo#f;pRNFNZsr7NUf^iq*xl3AwqiWdQ+h#|7PwK5KrQAQ&yl^wiCQE63PCt$J|k` zu%4vO(y9;=?ou{uz8br{8go^MMq(oVU>ns~IdGM%e@+lW){%y;^SGHVH#?(9!<4MK zBS_WIuKl@~tTC=BFs?#}bpu3;Rvx=X9(TL0lGz?8qvgjX0|ms?tfil%gmSPiq58^u zt#_#2H))B{lH;jOQ(N-G%5?>2J?|w+Zv`)j^wEPXuVW?0MmjJ|nBXJ!a>z$^zK_|i zzMQW{?fmaepW)_#7B_cf2Y&7RA6ARp@hH=A)6eT1Qyxj&4Ut!J-*OKGx{Y0`l(1nF zzZ&~A+N_@|!g8ib)-+PN7Yfaxrg=I8!7$1eAzi%N4)G*jFmf zZIXYy*yB;BL6oiaL5YD=F-+E7sU|SCsV$b3j>l~>;`e|IP9)|}^Q+#{rrSyPX_r<= zV^c|U2)OW;Vm!4RC~=z9!w{s?pbAZ zN;TdM(ylowbtz>!YaSXlCx{j~#UeQd{%YYTV;Uc$WBtW0`7r{eI5P5e{Tk&+f~CjD z!s>QCf!EVupz@NSj<)ZLfHAGG+sehH_MHDUyVdUBZcFxdWd4PrVdu4MS6gn%{n*** zjoW?kc-}TmY58=Yy-$q{BU$r83$u2k>hkq?MFQZ5;5RuFk{Tjf%fJn&&jjHQ&O}N= z$T&`jY=VptO-O`fjT01u!R8kndr?murxbb{HPPs(z!eaQ8NIu#`N3>D&k3&J`5i1i1i zoa@W!$+<$X9A6pSjB_NSgGJgqtVR18(#)eS z{Ru;Y9_bL}uQ*&7wF#~cCQLfAbK4=!$}Dl~Zp7isIwXcI{IELBq+5bit$vZ-PsH!P zC(J?_)z{eqId-cHMe$b2ITs``P7c2r+^>o9{i3VmPPs@l>A=)(V^j4iZJT&hupwxl z7t_Dd!&WVMaT7=-U)2*(B}Y3ORVA!_CDb&!=$w}od9!5M5nm0{m}2Z~-e+|$#FfNX zdZb<}Vr;gEW34`XK}eoFDOJnVvJgKqO;UyD)5{Sza^=@|vt0dbw5Mxb&_8eVKm~lK zL)}>gDUHvXz@tu$TcXM9W6(Cjn{PEOnUQuIf}auhH>uvsM@`hf=yD2bx>ZkGiDk>A zP}_3g@pck>v-ZpGe<0PG&C4Y577TngYaf-lTPrfggn(8pJvP2v)XOZ&CwKnQ$8Hly z!(#w9rDV7Q+N{zQsckU9gm}!dXkmQZXix&Uj{xw2dZkE#dQFXic==d?A{&Sg#8K`> zfsQFsB8(}z0i5VDAon$B5yl#SEJG)dMI{*j?m ztUVbcV{avsU=5_%z%F5kl3ep&*~{0QQO0AyOqG^a6ZXxNN>MJ!_K+%SO4(#GrX2qX z63Yn{%Xq~K7M%%@1W5uVXRuOg$AV3`FCaMq5Yq_G%V7O89~LE`q&1`@TNe*PFzb63 zl7wL>`rnFk09RHLB$pOr-;;yONzL$wcn%wPd-v22WqN84%i9oC8II18^54V>D?H5N zQE02grkpZw%Q2sPty%vGE< zaJRB0?KoCsU7GQfxw7#1jb_@efnS02I7^>ZA@trcm)l5`9wuJGbQC8%b{Tfpo-Jbrrk z*SGxF{ZGSaLINTzLTMEEidZrj#Eo5S`P2js_gs(famLm9bP z!hCQ01W)ASUX|Y_vUaj#x=2z+Jn1c?{ZZ* zx@!kPh$Gy{^0ry+T2t^;Wu9|Y>#4TC!V^)nxb~u2Lb7y_;ZL7)RVDdJf>sL(76F)$&lhkdtl8lfmz|4KGd%+r3e`R z=X>KD?6W3q0>6f}1A{yFBZwk8eU^EhS4W`l)|Lj;oJK-xRpi~)8!IhxcV40iap1=T za;mY6d5?E5M;w@WyLqXSqaDBKo4G5DtyQ1aknTiR#<>j1<**C|xY{0-fRMas^ZO!u zqaJ1VCJG`whdOVeMDF8quC8^Gxb9|Fopv%I#(ubqQl9g#Z_Z5iO}V(IHeOn(Jkkrg z>Iqw3Gllkq(MTQ`_P}ar@TR^&>Vd~eH4+_>-N-by0zF_ORP2097WxuOLAC{Nu?GFr zI;R=cg`@z9?o7Cp3$ue(Vak@ytTY0J{P}CWorXUJ+#gF@ui!$}g%8xfG)MuFn5O{0 zKm~gOM5?q9-+<}>;8_fs0><8sUywp{R3rg71t}mb(13q-yoCrPD%T)9RtzA5LNE9W z@XP5v=n|taSsrL7vuqAWr$^bISy~whbTU8rzFxjzxw{OxlsoOb;O!wbcjG%MMyqDN zKi6>*7*hVXLUtinwj>LiJgmYykzHi7ozPXj%NM$5lE>2GZ!i7jC+O}avODmk;^l3P z=w7Bdh2-TOm&+O&Qf6&p;^3W2$_Jw{u_eYpg7Vbn#4^~U!4N=>wj{=zm1BTBMuitb?(x;OtR<2$&zC)V zJItJsgTymrUpAFQR)jL(e6!&c2w0V1N;BrPziI?iOJ?#TR=Y!Lz&#lW;6jN&3*=%s zMT<&^n=v80vROfg0rQ1YPAzs8gIBb535HuJ zKqlAUJ)p993q%>LY)f4Rc)SKZz~jZPx8Q`sMq(y3;H@M#=xNc2WzmCKW(wd1c&_WR zLF710FF|j!Di@E?MP>U$9vrR2Q(kax?*~0Inwq^OLSOz(b3~KJ-!3iUx$Gm zCCPyu%SM7dht%U=jYruQ?*jmq?=u0nk>8wPmWo0SuEYC#-V4Axw8`ANr1UE&SuCj?E5;5Yk}{Xo@aHh^JmZdvsSJA??z0%hJVIWovk`S z`39#E?QSii;j18cM4>^;k+0*sIqZ(=>-F>bp{?wE`!bBuDZxjM9_#0c5EbbivJ*g# zUD>t>dEX6)QjV=E-h>nE4sc`B1*}1|(23v%xzT}aqH8*W0eeqv*se7L4rSZ8ZKmGr z4%a?i9@om@U!4BJb$la(?of9FR69p97i=k=P!?^KHcx8>`%Bwm$-}(fU}?CeyU))7 zge`q`AcnDs`w~ueQEoy>MT4IQIrrdv=5>t1s|YHTL99WN=OfC~WQC&cEgm$)NydG! z&#AqyszrHtUkz%^wI+VyKo(4tUrBhlq3*43`+t2rEy)YC+S$zsd^(zDsq2+?)&|p1 z)U{5y3dgYh;fU3E0S>E}dg1h@c`&ppwFn`I*%;no9JyO@jivv^Qn3+H^tXNy#9;eN z>KPnPOZ)sP@lFuE(#v{%%0RYXx6#pMNIc!wPD=vV?D|+VRSw2zy>B-N_~HN2mQo<4zRH28vp zE!X!Ph&}UUz?CE|8g{qFxXT~0kF+i}fu%4PCIhaB$K9sJ07b%yQOm~b|C=nB1Si4y z!6*hwccT{@G0Z&u$i#`|GT+rs--99j+LgclU}bIls$o59`BRT_3FM3JnMMW#4s8Hf zI)Ks7d>Q>yI^4ckl%Ni2{I<)r!qsF--efEQjh@Dkr&cQea5Z1f+E-L>FFRW*+0chm zesxQ6nN(K!Y8Yh_pl<)nzz*?KkV}-TsIJ}``|0v{8$6HV@2*m_d6IWwM;XhJP2K&` zqn4JTP#9_FiQa4y992B4rDRaOktt0{PbX2?GN8or=V5%Nu}H2gDpZK(NuGwT)@VV7 z25C?+ZTa%*qL`?GTTKhUvXq+=&PI^!NuG_aR%k(njiOb*AnUC>D>dkYLwOp^DNA*d z_304;zrj*XOOEUObjDhbG7whUZ`aIQ1sHTO3AfM}H7`yws3cSYmP#<-uR;r-T2MN# z0Mj)jrLgehTbH=5?|HNLn02Z)wimf4SXB8;M5I4i-Do~trz>s!h#_6cO0TSrLZL_3 zIm^n%bSpG4No+yjEK(|MAA^R408Du^i_z0^WR8#aLBz>easeB2AzSG-b9K&onG+@# zrQhMo{m`0i*J(P*y=cngr(qmbB)m%JPxRt;(bVD{jiQvL4B9rdW09Z8iEKX=Kkp;o z1~8K*x8#_s;V6|~`hGZ$r}!{tkc$<2>;|TRQ(SQzof!{3%lg#%d~42%Q<;xhyf<9* z+EEBRbS{xs$mT0t&g=3BV%T_eiLF?%wO+}NwLQ1eTfaxBg0q>=!+pfYvo(UaBS&Ds z+w=D*dlj35^C%I>GEvMFE%FQz<3b>o z^&M@t7;9WJdj_3~O+Wj(A;M_&sa7v7PqMJ>S$6zl{`aD`T4Mf*hEE@L6y<37M5j$t z(S6taAM@KkN|Xo(I$mv50=`SB(v#5DDSh`g^3ps-@vH8Y*(NFM6XlzY?@M8~+R59F z{=@2?%ydG<^eNS~nU-YvZp{Va(=2Ild(9ZGhDYH<3!L8II5?IDn{$gAZu5+!n z-}xzyWivJ-zaJmX^%m?cyLIvE-QdVa-|ph)BfV`#V^wi>b^s@G?5Bn@gc+68irx}^ z)k{*B5#gNZ;+Gx@Go#tZJBP;7nTT~(Tj}z)YVAL91HQv&5lG9CM-8=l-QUK@SGC&2 z){s`%Z|inyfW zH(F@7PRqr}Yqz1fSalO{nn(B_|HU%)@eK4n(tX(Vk%9cMV|q?sI*Vg^)qQx+x!1d= z;@e?&mwm~6Q^{w0gZJqhtdoznvxZSQkZczLnKGTMicH>q`L6ca=(jN*%$F9Ax}&)855_UB=@xcx{jRmMR6ky8 zqkO%mI>*lp*#&^;5EOcdO$2zKh^{62I1p^*w7TeQ7NM2+9JKb`qPx%oK#}Sgd~Y6y zzrk~si_gj@G}-9Ez6MpCY1887HTs_(YWuVP79cAvy_PnfY@&g)_EHzW<1Tu}Q_?l#)XRfs>r=rWo1KoaUsU2Av}kF+*?D)bW8doH<0cfrqi zVPPTYvbsw~yNCEVSx*^mCq%qHmp|v)y?n;bc;X%@7l{^?*5P_4rT9Le?>a~Sf%0Q9 z1-{zfBT&lU^2R&XoHmk43puBj*_13xuC&ByVu4fz75@ml-ceN^l$j0Q*H(L>6#-^8 zJ!Y03$38C601}(J&bOa89ddA33xzW6YD|F~5E3bvp+OIc$GQ*4pUTW4k6o)VV7@ym zF@mV(nuWC|#bJ`_DKzwNazkCJDy~O!M2B`KziJ=OL94$#C%69=B1njd5V3k`Alij#J9&|cxMS!Y&c+t>p>exy&79O!Zn;8Ut< z&tCx(VMW+Tb#m+PA<76}?K8`&drUeHzBZm{WU0C%Y0c>~+4j(%zrEFM7-}54o_*Z& zx-ycj_WfK!dr-qpv1ec7hU*C(@za!PeV-T~xYWyq`^kfHBnc$2I7Owhr;k5Kgk3Z( zQsIA-3Lt5RvVXf(#eAj(l~j;@iqChV_ZoBB1tN4NPzv zT*AMNF10y8c=gC%c0*Vj%KL{yKZM|4A95 zu}t7|V1^7>`~J!&_E0Q@SOzE%E~P_Wjy6W4K(=8~qi(xvgV28UM1!P$NkxIHe4YS7 z&!C>NQ_o|BTtglD>*#K`?ue>^3<3TZY_&>1nL2$pTPbNVv%d3wlma~U56$egKTB6L zGzEyDqTqv%4<6*fAnppmO`NUf;`VI%^Yq4WdU+wnFKGd2w15NPU_pZ_} zVfe44mSct6l8C-r6#(x0>FGiy(|9gA)BtiXi6SOUu*V1=A%pm_bf`yYgZK&t?+{6u zbzcBzWP@z%p4+=}?4|9KT6-B;yh2~8)?$6$>IHlR`Mxb8kI5sNGOJ!pNZb_H%uWgy-3KB;fyzHBWNfiBhh(ev~@2T`jbZzddjPkLc_+UiWNYS5i-W43&;X$ zYedu{4KwHhDiGldGbsKW;m5rC)_=2P><0fA!Y9t4{x`z^ISfSjv;lDcLiol-|3dg# zk^e^c^D_S+{IbY@A$)@@>i>f9fBt_Ud^q?T@QwjkMo1k4IvP}mz*2k5avcMC8DNvB zwogJuJ@y#X7;B&g?11CDg<3;wXZR0RLWS{NvG8FeHC?QL?2`fN!PCneK@xWf%r@ar z7rqRB0X-Pn+Lr}=7!4uXil4=c1@pLKABzLsqlgwii(3nnWC*QxHlYj>rqUrH$01cG zzqNJXRUJf^6l(dYFW$zJC-Nrow(M528SQRIoZX0@AS4OQL-`pZbhk&s(W1+z=Z2sX zn=g~5crYk`)lgX>B&(MUyTHr1T6gVdU#mc#h`kpr zGGFQeOCZxE_d$){x4Ggb$rA`=*u4z2$&xb@P66vS|50)ExME>nbnjbRVPu0z_RC) z`39|>!}LBbjG)=N4Bs?xOjILOq(PiLT0t=s+qX=w5 zp=eQo=W>jksm><5%CP!G``DzUFIceX!ONbRDD|) zT6>$GLALL`PtXtXVuA2WhZomoaQBu)#Kb`@z_88Omc_EfPS(cqugOm`Dg%cz=8lC- z58NJ!nE{tFh|JFAkWn`D$v*psCVUsz))9LtyY#raE!LStnbF<5*g=hCd_CHurpWO5 zGb#D}ae3*^pcLnm)x7t6O?5^F;{)Z_`DQ+b3l)B|?NYlW#rdf`x6xZ|ya{_*ztNAt z_ZE5qh0cULzB9A@@9K)5%Vs(Z6Fz(tRGXIUUcZ`lSO8b6e z+a24sZQHihv2ELSI<}LJZFkV=7#&-ux@YFhyw5Y|{cyh2N@~@r+I45G-1~R^vGu!; z@AVS`S5#iD8^`s)k~}n>omlhRhC)ui;BE@&+E%jcwaRWTR9>8}G?Vm`U2_W;b6<;g zEZaOn)9B9LYsb~J5Y$`k>lbfcxP=D;zxuar%6)!*+u`dw7C3_s!+h7T+dMTw`~MP{qIB)9 zmLDB{yr`77_*i?4sNTRJ!?3GvU5{jPoOBD~w`&^|6n+zd?8Evs3HyWRvI_Iag!$Or zd!Vmb-`!{avg+ulU ztyV8-@&w#0#u;?3VS0c}x{T!LQ0Fo-1+&7oag1*67j=jD-rof)L9cyBfmjOOm`bEB z_bAW;aT}{^114K@mGa$gyWc_;Um^LD5FrCg^MuuzNoz|LkXl(?Y`bJYC7?W-jwo8I zlW_L19z=L6atci1Gu(&ZA>htYO5SEC`?;FCYHEo+i_D%s|7?aBbkdzq7L+c!pz;We zBrq>(=kzX*)!UkBT++0c|f}Iw<%I8ukP82?$Ljxl$d=0CCZx!_Te>xZc(XWFF@yE+{{xHkhra zfXnn$7eC`Fy}H|Cs~O`&rX9mB(8X$^!F$xfnR{s1(qe2}cMGR~`*KNS*`|#1@IvWC zch0ZsFjk$Ik{1*QN+UpCCZb{I=N!~qAmE#w^t3WK4*8186W7ke4|3=$cTF~LJdc8j z&?^}UbESdDD&rrb$dV4ZEYK}j*w_42Gm=~}1W5<=sh%N;B3G`=tYBu1OdZ+&)&8P= ziX3?Si*#Qr-RVYHh=I}wNoI`Y=o2fb2HI+`LEcYL1-lv#Yne6&YSTvWB5lGdI*j0p z>>Kd2M&INv>WiYrjTO0SyWyo{xzxOzBP3rm6E;8Q52@ZhAE%X2g(P0X{A#@m3;g@* zo8TMXmtVooQMfv=GhD4BZjwoKQCFt($IkE@v{ao(S+<#Xrs*lFafs@URm$KxAJk26 z$@xW9Fv)9@a;|TP6MMm~H-#uV{q}%Omy?@6ny@N-Ah+pq;R_TZPE}2osNeg`s`AZy z&jevO-Vz}NuJd=9?4^rPVfm-VIclz-Vt)9o3Y)*`U642Itu;G^+kOnIB8L>X^kRXP z-hw|`JvNhvY`Vy}LO+Z-aW|LAcO&O^u$My2iz7I>^1wAfQ|9Wnk>}%WmJXRokwO$+ z2%KNf+E#YOVnhRx!Z!0cGv@ZZ6}SxT;Gx^NcUshewX=k7aUpj<&PtGAs-T$l`tgoK z+$)yv!=ImgT@_$gS~Wc6{*Eq&Sz*{M^<75H)wQa6P19)FEEGNQbb-BCK3Dy_pwYEN z+ZF>B6Y(`9GLs@4^E#Z-{s@WCWjhI-`B;df;>VT{C*`Rx=+A=v-=y~uMyg365sH